testmcpy-oauth-probe 0.1.0__py3-none-any.whl

testmcpy_oauth_probe/__init__.py

@@ -0,0 +1,44 @@
+"""Typed public API for the headless OAuth/MCP probe."""
+
+from testmcpy_oauth_probe.config import (
+    ConfigError,
+    load_manifest,
+    loads_manifest,
+    manifest_json_schema,
+    report_json_schema,
+)
+from testmcpy_oauth_probe.models import (
+    CONFIG_SCHEMA,
+    REPORT_SCHEMA,
+    AuthFlow,
+    CapabilityPolicy,
+    CheckResult,
+    CheckStatus,
+    Manifest,
+    ProbeReport,
+    RunReport,
+    TargetConfig,
+)
+from testmcpy_oauth_probe.runner import ProbeRunner, run_manifest
+from testmcpy_oauth_probe.transport import HttpTransport
+
+__all__ = [
+    "CONFIG_SCHEMA",
+    "REPORT_SCHEMA",
+    "AuthFlow",
+    "CapabilityPolicy",
+    "CheckResult",
+    "CheckStatus",
+    "ConfigError",
+    "Manifest",
+    "HttpTransport",
+    "ProbeReport",
+    "ProbeRunner",
+    "RunReport",
+    "TargetConfig",
+    "load_manifest",
+    "loads_manifest",
+    "manifest_json_schema",
+    "report_json_schema",
+    "run_manifest",
+]

testmcpy_oauth_probe/__main__.py

@@ -0,0 +1,3 @@
+from testmcpy_oauth_probe.cli import main
+
+raise SystemExit(main())

testmcpy_oauth_probe/cli.py

@@ -0,0 +1,152 @@
+"""Minimal argparse CLI, independent of testmcpy's UI/LLM dependencies."""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import sys
+from collections.abc import Sequence
+from pathlib import Path
+
+from testmcpy_oauth_probe.config import (
+    ConfigError,
+    dump_manifest_schema,
+    dump_report_schema,
+    load_manifest,
+    loads_manifest,
+)
+from testmcpy_oauth_probe.models import CONFIG_SCHEMA, Correlation
+from testmcpy_oauth_probe.reporters import to_human, to_json, to_jsonl, to_junit
+from testmcpy_oauth_probe.runner import ProbeRunner
+
+# `--config` stays first so it remains the name in `--help` output and in every
+# existing pipeline; `--manifest` is an accepted alias. argparse stores both
+# under `dest="config"`.
+_MANIFEST_FLAGS = ("--config", "--manifest")
+_MANIFEST_KWARGS = {
+    "dest": "config",
+    "metavar": "PATH",
+    "help": "Path to the versioned probe manifest (YAML or JSON)",
+}
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="testmcpy-oauth",
+        description="Vendor-neutral OAuth/MCP interoperability probe (not formal certification)",
+    )
+    subparsers = parser.add_subparsers(dest="command", required=True)
+    validate = subparsers.add_parser("validate", help="Validate a manifest without network access")
+    # Everything else — the docs, the schema name, the Auth Smoke page, this
+    # tool's own help text — calls the file a manifest, so `--manifest` is the
+    # flag people reach for first. Accept both rather than make them guess.
+    validate.add_argument(*_MANIFEST_FLAGS, required=True, **_MANIFEST_KWARGS)
+    schema = subparsers.add_parser("schema", help="Print a versioned JSON Schema")
+    schema.add_argument("--kind", choices=("manifest", "report"), default="manifest")
+    schema.set_defaults(command="schema")
+    check = subparsers.add_parser("check", help="Run configured targets headlessly")
+    check.add_argument(*_MANIFEST_FLAGS, required=True, **_MANIFEST_KWARGS)
+    check.add_argument("--target", action="append", dest="targets")
+    check.add_argument("--profile")
+    check.add_argument("--format", choices=("human", "json", "jsonl"), default="human")
+    check.add_argument("--output", default="-")
+    check.add_argument("--junit")
+    check.add_argument("--run-id")
+    check.add_argument("--service", help="Report label only: recorded in correlation.service.")
+    check.add_argument("--region", help="Report label only: recorded in correlation.region.")
+    check.add_argument(
+        "--revision",
+        help=(
+            "Report label only: recorded in correlation.revision and never "
+            "compared against the deployed revision. This is not a revision assertion."
+        ),
+    )
+    check.add_argument(
+        "--deployment-id", help="Report label only: recorded in correlation.deployment_id."
+    )
+    discover = subparsers.add_parser("discover", help="Run discovery without credentials")
+    discover.add_argument("--url", required=True)
+    discover.add_argument("--format", choices=("human", "json", "jsonl"), default="human")
+    return parser
+
+
+def _write(path: str, content: str) -> None:
+    if path == "-":
+        sys.stdout.write(content)
+    else:
+        Path(path).write_text(content, encoding="utf-8")
+
+
+def _render(report_format: str, report: object) -> str:
+    from testmcpy_oauth_probe.models import RunReport
+
+    assert isinstance(report, RunReport)
+    if report_format == "json":
+        return to_json(report)
+    if report_format == "jsonl":
+        return to_jsonl(report)
+    return to_human(report)
+
+
+async def _run(args: argparse.Namespace) -> int:
+    if args.command == "discover":
+        manifest = loads_manifest(
+            f"""schema: {CONFIG_SCHEMA}
+targets:
+  discovery:
+    mcp_url: {args.url!r}
+    oauth:
+      flow: none
+      # `discover` is advertised as read-only reconnaissance. error_probe
+      # defaults to true and now applies to every flow, so opt out explicitly
+      # rather than POST an unsupported grant to someone's token endpoint.
+      error_probe: false
+"""
+        )
+        report = await ProbeRunner().run_manifest(manifest)
+        _write("-", _render(args.format, report))
+        return report.exit_code
+    manifest = load_manifest(args.config)
+    report = await ProbeRunner().run_manifest(
+        manifest,
+        target_ids=args.targets,
+        profile=args.profile,
+        run_id=args.run_id,
+        correlation_override=Correlation(
+            service=args.service,
+            region=args.region,
+            revision=args.revision,
+            deployment_id=args.deployment_id,
+        ),
+    )
+    _write(args.output, _render(args.format, report))
+    if args.junit:
+        _write(args.junit, to_junit(report))
+    return report.exit_code
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    parser = build_parser()
+    args = parser.parse_args(argv)
+    try:
+        if args.command == "schema":
+            print(dump_report_schema() if args.kind == "report" else dump_manifest_schema())
+            return 0
+        if args.command == "validate":
+            manifest = load_manifest(args.config)
+            print(
+                f"valid {manifest.schema} manifest: {len(manifest.targets)} target(s), "
+                f"{len(manifest.profiles)} profile(s)"
+            )
+            return 0
+        return asyncio.run(_run(args))
+    except (ConfigError, ValueError) as exc:
+        print(f"configuration error: {exc}", file=sys.stderr)
+        return 2
+    except KeyboardInterrupt:
+        print("interrupted", file=sys.stderr)
+        return 2
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

testmcpy_oauth_probe/config.py

@@ -0,0 +1,426 @@
+"""Strict loader for the versioned OAuth probe manifest."""
+
+from __future__ import annotations
+
+import json
+import math
+import os
+import re
+from collections.abc import Mapping
+from importlib.resources import files
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+from testmcpy_oauth_probe.models import (
+    CONFIG_SCHEMA,
+    SUPPORTED_SPEC_PROFILES,
+    AuthFlow,
+    CapabilityPolicy,
+    ClientAuthMethod,
+    Correlation,
+    Expectations,
+    Manifest,
+    OAuthConfig,
+    RunProfile,
+    SecretRef,
+    TargetConfig,
+    ValueRef,
+)
+
+
+class ConfigError(ValueError):
+    pass
+
+
+_ENV_RE = re.compile(r"^\$\{([A-Za-z_][A-Za-z0-9_]*)(?::-(.*))?\}$")
+
+
+def _mapping(value: Any, path: str) -> dict[str, Any]:
+    if value is None:
+        return {}
+    if not isinstance(value, Mapping):
+        raise ConfigError(f"{path} must be an object")
+    return {str(key): item for key, item in value.items()}
+
+
+def _only(data: Mapping[str, Any], allowed: set[str], path: str) -> None:
+    unknown = sorted(set(data) - allowed)
+    if unknown:
+        raise ConfigError(f"{path} contains unknown field(s): {', '.join(unknown)}")
+
+
+def _expand_env(value: str, path: str) -> str:
+    """Resolve a whole-scalar `${NAME}` / `${NAME:-default}` reference."""
+    match = _ENV_RE.match(value)
+    if match is None:
+        return value
+    env_name, default = match.groups()
+    resolved = os.environ.get(env_name, default)
+    if resolved is None:
+        raise ConfigError(f"{path} references unset environment variable {env_name!r}")
+    return resolved
+
+
+def _string(value: Any, path: str, *, required: bool = False) -> str | None:
+    if value is None and not required:
+        return None
+    if not isinstance(value, str) or not value:
+        raise ConfigError(f"{path} must be a non-empty string")
+    return _expand_env(value, path)
+
+
+def _strings(value: Any, path: str) -> tuple[str, ...]:
+    if value is None:
+        return ()
+    if not isinstance(value, list) or not all(isinstance(item, str) and item for item in value):
+        raise ConfigError(f"{path} must be an array of non-empty strings")
+    # Array elements expand exactly like scalars. Without this, `mcp_url:
+    # ${SMOKE_MCP_URL}` resolved while `issuers: ["${SMOKE_ORIGIN}"]` was
+    # compared literally, which forced a manifest generated per run for any
+    # ephemeral target. Duplicates are judged after expansion, since two
+    # references that resolve to the same value are the same expectation.
+    expanded = tuple(_expand_env(item, f"{path}[{index}]") for index, item in enumerate(value))
+    if any(not item for item in expanded):
+        raise ConfigError(f"{path} must not contain empty values after expansion")
+    if len(set(expanded)) != len(expanded):
+        raise ConfigError(f"{path} must not contain duplicates")
+    return expanded
+
+
+def _secret_ref(value: Any, path: str) -> SecretRef | None:
+    if value is None:
+        return None
+    data = _mapping(value, path)
+    _only(data, {"env"}, path)
+    env_name = _string(data.get("env"), f"{path}.env", required=True)
+    assert env_name is not None
+    if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", env_name):
+        raise ConfigError(f"{path}.env must be an environment variable name")
+    return SecretRef(env=env_name)
+
+
+def _value_ref(value: Any, path: str) -> ValueRef | None:
+    if value is None:
+        return None
+    if isinstance(value, str):
+        return ValueRef(value=value)
+    data = _mapping(value, path)
+    _only(data, {"env"}, path)
+    env_name = _string(data.get("env"), f"{path}.env", required=True)
+    assert env_name is not None
+    if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", env_name):
+        raise ConfigError(f"{path}.env must be an environment variable name")
+    return ValueRef(env=env_name)
+
+
+def _policies(value: Any, path: str) -> dict[str, CapabilityPolicy]:
+    data = _mapping(value, path)
+    result: dict[str, CapabilityPolicy] = {}
+    for name, raw_policy in data.items():
+        try:
+            result[name] = CapabilityPolicy(raw_policy)
+        except ValueError as exc:
+            choices = ", ".join(policy.value for policy in CapabilityPolicy)
+            raise ConfigError(f"{path}.{name} must be one of: {choices}") from exc
+    return result
+
+
+def _oauth(value: Any, path: str) -> OAuthConfig:
+    data = _mapping(value, path)
+    _only(
+        data,
+        {
+            "flow",
+            "access_token",
+            "refresh_token",
+            "refresh_token_disposable",
+            "authorization_code",
+            "pkce_verifier",
+            "client_id",
+            "client_secret",
+            "client_auth_method",
+            "scopes",
+            "resource",
+            "audience",
+            "token_endpoint",
+            "redirect_uri",
+            "error_probe",
+        },
+        path,
+    )
+    try:
+        flow = AuthFlow(data.get("flow", "none"))
+        auth_method = ClientAuthMethod(data.get("client_auth_method", "none"))
+    except ValueError as exc:
+        raise ConfigError(f"{path} contains an unsupported flow or client auth method") from exc
+    error_probe = data.get("error_probe", True)
+    if not isinstance(error_probe, bool):
+        raise ConfigError(f"{path}.error_probe must be a boolean")
+    disposable = data.get("refresh_token_disposable", False)
+    if not isinstance(disposable, bool):
+        raise ConfigError(f"{path}.refresh_token_disposable must be a boolean")
+    config = OAuthConfig(
+        flow=flow,
+        access_token=_secret_ref(data.get("access_token"), f"{path}.access_token"),
+        refresh_token=_secret_ref(data.get("refresh_token"), f"{path}.refresh_token"),
+        refresh_token_disposable=disposable,
+        authorization_code=_secret_ref(
+            data.get("authorization_code"), f"{path}.authorization_code"
+        ),
+        pkce_verifier=_secret_ref(data.get("pkce_verifier"), f"{path}.pkce_verifier"),
+        client_id=_value_ref(data.get("client_id"), f"{path}.client_id"),
+        client_secret=_secret_ref(data.get("client_secret"), f"{path}.client_secret"),
+        client_auth_method=auth_method,
+        scopes=_strings(data.get("scopes"), f"{path}.scopes"),
+        resource=_string(data.get("resource"), f"{path}.resource"),
+        audience=_string(data.get("audience"), f"{path}.audience"),
+        token_endpoint=_string(data.get("token_endpoint"), f"{path}.token_endpoint"),
+        redirect_uri=_string(data.get("redirect_uri"), f"{path}.redirect_uri"),
+        error_probe=error_probe,
+    )
+    required: dict[AuthFlow, tuple[tuple[str, Any], ...]] = {
+        AuthFlow.BEARER: (("access_token", config.access_token),),
+        AuthFlow.REFRESH_TOKEN: (("refresh_token", config.refresh_token),),
+        AuthFlow.AUTHORIZATION_CODE: (
+            ("authorization_code", config.authorization_code),
+            ("pkce_verifier", config.pkce_verifier),
+            ("redirect_uri", config.redirect_uri),
+        ),
+        AuthFlow.CLIENT_CREDENTIALS: (("client_id", config.client_id),),
+    }
+    missing = [name for name, item in required.get(flow, ()) if item is None]
+    if missing:
+        raise ConfigError(f"{path} flow {flow.value!r} requires: {', '.join(missing)}")
+    if auth_method is not ClientAuthMethod.NONE and config.client_id is None:
+        raise ConfigError(f"{path}.client_id is required for {auth_method.value}")
+    if auth_method is not ClientAuthMethod.NONE and config.client_secret is None:
+        raise ConfigError(f"{path}.client_secret is required for {auth_method.value}")
+    if flow is AuthFlow.CLIENT_CREDENTIALS and auth_method is ClientAuthMethod.NONE:
+        raise ConfigError(f"{path} client_credentials requires confidential client authentication")
+    return config
+
+
+def _expectations(value: Any, path: str) -> Expectations:
+    data = _mapping(value, path)
+    _only(
+        data,
+        {
+            "issuers",
+            "token_issuers",
+            "resources",
+            "audiences",
+            "scopes",
+            "grants",
+            "auth_methods",
+            "endpoints",
+            "capabilities",
+            "unauthorized_status",
+            "initialize_status",
+            "initialized_statuses",
+            "tools_list_status",
+            "min_tools",
+        },
+        path,
+    )
+    status_fields: dict[str, int] = {}
+    for name, default in (
+        ("unauthorized_status", 401),
+        ("initialize_status", 200),
+        ("tools_list_status", 200),
+    ):
+        raw = data.get(name, default)
+        if not isinstance(raw, int) or isinstance(raw, bool) or not 100 <= raw <= 599:
+            raise ConfigError(f"{path}.{name} must be an HTTP status between 100 and 599")
+        status_fields[name] = raw
+    min_tools = data.get("min_tools", 0)
+    if not isinstance(min_tools, int) or isinstance(min_tools, bool) or min_tools < 0:
+        raise ConfigError(f"{path}.min_tools must be a non-negative integer")
+    initialized_statuses = data.get("initialized_statuses", [200, 202])
+    if (
+        not isinstance(initialized_statuses, list)
+        or not initialized_statuses
+        or not all(
+            isinstance(item, int) and not isinstance(item, bool) and 100 <= item <= 599
+            for item in initialized_statuses
+        )
+    ):
+        raise ConfigError(f"{path}.initialized_statuses must be a non-empty array of HTTP statuses")
+    if len(set(initialized_statuses)) != len(initialized_statuses):
+        raise ConfigError(f"{path}.initialized_statuses must not contain duplicates")
+    return Expectations(
+        issuers=_strings(data.get("issuers"), f"{path}.issuers"),
+        token_issuers=_strings(data.get("token_issuers"), f"{path}.token_issuers"),
+        resources=_strings(data.get("resources"), f"{path}.resources"),
+        audiences=_strings(data.get("audiences"), f"{path}.audiences"),
+        scopes=_strings(data.get("scopes"), f"{path}.scopes"),
+        grants=_policies(data.get("grants"), f"{path}.grants"),
+        auth_methods=_policies(data.get("auth_methods"), f"{path}.auth_methods"),
+        endpoints=_policies(data.get("endpoints"), f"{path}.endpoints"),
+        capabilities=_policies(data.get("capabilities"), f"{path}.capabilities"),
+        unauthorized_status=status_fields["unauthorized_status"],
+        initialize_status=status_fields["initialize_status"],
+        initialized_statuses=tuple(initialized_statuses),
+        tools_list_status=status_fields["tools_list_status"],
+        min_tools=min_tools,
+    )
+
+
+def _correlation(value: Any, path: str) -> Correlation:
+    data = _mapping(value, path)
+    _only(data, {"service", "region", "revision", "deployment_id"}, path)
+    return Correlation(
+        service=_string(data.get("service"), f"{path}.service"),
+        region=_string(data.get("region"), f"{path}.region"),
+        revision=_string(data.get("revision"), f"{path}.revision"),
+        deployment_id=_string(data.get("deployment_id"), f"{path}.deployment_id"),
+    )
+
+
+def _target(target_id: str, value: Any, defaults: Mapping[str, Any]) -> TargetConfig:
+    if not target_id:
+        raise ConfigError("target IDs must be non-empty strings")
+    path = f"targets.{target_id}"
+    data = _mapping(value, path)
+    _only(
+        data,
+        {
+            "mcp_url",
+            "spec_profile",
+            "correlation",
+            "oauth",
+            "expectations",
+            "timeout_seconds",
+            "max_response_bytes",
+            "transient_retries",
+            "allow_http_loopback",
+            "allow_private_network",
+        },
+        path,
+    )
+    merged = {**defaults, **data}
+    mcp_url = _string(merged.get("mcp_url"), f"{path}.mcp_url", required=True)
+    spec_profile = _string(merged.get("spec_profile", "mcp-2025-06-18"), f"{path}.spec_profile")
+    assert mcp_url is not None and spec_profile is not None
+    if spec_profile not in SUPPORTED_SPEC_PROFILES:
+        raise ConfigError(
+            f"{path}.spec_profile must be one of: {', '.join(SUPPORTED_SPEC_PROFILES)}"
+        )
+    timeout = merged.get("timeout_seconds", 20.0)
+    if (
+        not isinstance(timeout, (int, float))
+        or isinstance(timeout, bool)
+        or not math.isfinite(timeout)
+        or timeout <= 0
+    ):
+        raise ConfigError(f"{path}.timeout_seconds must be positive")
+    max_bytes = merged.get("max_response_bytes", 1_048_576)
+    retries = merged.get("transient_retries", 1)
+    if not isinstance(max_bytes, int) or isinstance(max_bytes, bool) or max_bytes < 1024:
+        raise ConfigError(f"{path}.max_response_bytes must be an integer >= 1024")
+    if not isinstance(retries, int) or isinstance(retries, bool) or not 0 <= retries <= 5:
+        raise ConfigError(f"{path}.transient_retries must be between 0 and 5")
+    for boolean_name in ("allow_http_loopback", "allow_private_network"):
+        if not isinstance(merged.get(boolean_name, False), bool):
+            raise ConfigError(f"{path}.{boolean_name} must be a boolean")
+    return TargetConfig(
+        id=target_id,
+        mcp_url=mcp_url,
+        spec_profile=spec_profile,
+        correlation=_correlation(merged.get("correlation"), f"{path}.correlation"),
+        oauth=_oauth(merged.get("oauth"), f"{path}.oauth"),
+        expectations=_expectations(merged.get("expectations"), f"{path}.expectations"),
+        timeout_seconds=float(timeout),
+        max_response_bytes=max_bytes,
+        transient_retries=retries,
+        allow_http_loopback=merged.get("allow_http_loopback", False),
+        allow_private_network=merged.get("allow_private_network", False),
+    )
+
+
+def _parse(document: Any) -> Manifest:
+    root = _mapping(document, "manifest")
+    _only(root, {"schema", "defaults", "targets", "profiles"}, "manifest")
+    schema = _string(root.get("schema"), "schema", required=True)
+    if schema != CONFIG_SCHEMA:
+        raise ConfigError(f"unsupported schema {schema!r}; expected {CONFIG_SCHEMA!r}")
+    defaults = _mapping(root.get("defaults"), "defaults")
+    _only(
+        defaults,
+        {
+            "spec_profile",
+            "timeout_seconds",
+            "max_response_bytes",
+            "transient_retries",
+            "allow_http_loopback",
+            "allow_private_network",
+        },
+        "defaults",
+    )
+    target_values = _mapping(root.get("targets"), "targets")
+    if not target_values:
+        raise ConfigError("manifest must define at least one target")
+    targets = {
+        target_id: _target(target_id, value, defaults) for target_id, value in target_values.items()
+    }
+    profile_values = _mapping(root.get("profiles"), "profiles")
+    profiles: dict[str, RunProfile] = {}
+    for profile_name, value in profile_values.items():
+        if not profile_name:
+            raise ConfigError("profile names must be non-empty strings")
+        data = _mapping(value, f"profiles.{profile_name}")
+        _only(data, {"targets"}, f"profiles.{profile_name}")
+        selected = _strings(data.get("targets"), f"profiles.{profile_name}.targets")
+        if not selected:
+            raise ConfigError(f"profiles.{profile_name}.targets must not be empty")
+        missing = sorted(set(selected) - set(targets))
+        if missing:
+            raise ConfigError(
+                f"profiles.{profile_name} names unknown targets: {', '.join(missing)}"
+            )
+        profiles[profile_name] = RunProfile(targets=selected)
+    return Manifest(schema=schema, targets=targets, profiles=profiles)
+
+
+def loads_manifest(content: str, *, source: str = "<string>") -> Manifest:
+    try:
+        document = yaml.safe_load(content)
+    except yaml.YAMLError as exc:
+        raise ConfigError(f"invalid YAML/JSON in {source}") from exc
+    return _parse(document)
+
+
+def load_manifest(path: str | Path) -> Manifest:
+    source = Path(path)
+    try:
+        content = source.read_text(encoding="utf-8")
+    except OSError as exc:
+        raise ConfigError(f"cannot read manifest {source}") from exc
+    return loads_manifest(content, source=str(source))
+
+
+def _json_schema(filename: str) -> dict[str, Any]:
+    schema = files("testmcpy_oauth_probe").joinpath(f"schemas/{filename}")
+    value = json.loads(schema.read_text(encoding="utf-8"))
+    if not isinstance(value, dict):  # pragma: no cover - package invariant
+        raise RuntimeError("packaged schema is invalid")
+    return value
+
+
+def manifest_json_schema() -> dict[str, Any]:
+    """Return the packaged machine-readable manifest v1 schema."""
+    return _json_schema("oauth-smoke-v1.schema.json")
+
+
+def report_json_schema() -> dict[str, Any]:
+    """Return the packaged machine-readable report v1 schema."""
+    return _json_schema("oauth-smoke-report-v1.schema.json")
+
+
+def dump_manifest_schema() -> str:
+    return json.dumps(manifest_json_schema(), indent=2, sort_keys=True)
+
+
+def dump_report_schema() -> str:
+    return json.dumps(report_json_schema(), indent=2, sort_keys=True)