toolframe-testkit 0.1.2__py3-none-any.whl

toolframe_testkit/__init__.py

@@ -0,0 +1,25 @@
+"""Developer checks at Toolframe's public SDK, CLI and HTTP boundaries."""
+
+from .contracts import (
+    ContractCheckError,
+    ContractIssue,
+    ContractReport,
+    ResourceExample,
+    ServiceExample,
+    check_definition,
+)
+from .harness import ApplicationHarness, CLIResult, HarnessError, HarnessTimeout
+
+__version__ = "0.1.2"
+__all__ = [
+    "ApplicationHarness",
+    "CLIResult",
+    "ContractCheckError",
+    "ContractIssue",
+    "ContractReport",
+    "HarnessError",
+    "HarnessTimeout",
+    "ResourceExample",
+    "ServiceExample",
+    "check_definition",
+]

toolframe_testkit/contracts.py

@@ -0,0 +1,302 @@
+"""Static and concrete-example checks using the public SDK contract validators."""
+
+from __future__ import annotations
+
+from collections.abc import Iterable, Mapping
+from dataclasses import dataclass
+from importlib.resources import files
+from typing import Any
+
+from jsonschema import SchemaError
+from jsonschema import ValidationError as SchemaValidationError
+from pydantic import ValidationError
+from toolframe_sdk import (
+    SYSTEM_SCOPES,
+    PluginDefinition,
+    ServiceSpec,
+    load_object,
+    validate,
+)
+
+_ABSENT = object()
+
+
+@dataclass(frozen=True)
+class ContractIssue:
+    code: str
+    path: tuple[str | int, ...]
+    message: str
+    hint: str
+
+
+@dataclass(frozen=True)
+class ServiceExample:
+    service: str
+    method: str
+    input: Any
+    output: Any = _ABSENT
+
+
+@dataclass(frozen=True)
+class ResourceExample:
+    resource_id: str
+    input: Any
+    output: Any = _ABSENT
+
+
+@dataclass(frozen=True)
+class ContractReport:
+    definition: PluginDefinition | None
+    issues: tuple[ContractIssue, ...]
+
+    @property
+    def ok(self) -> bool:
+        return not self.issues
+
+    def assert_valid(self) -> PluginDefinition:
+        if not self.ok or self.definition is None:
+            raise ContractCheckError(self)
+        return self.definition
+
+
+class ContractCheckError(ValueError):
+    def __init__(self, report: ContractReport):
+        self.report = report
+        lines = [
+            f"{item.code} at {'.'.join(map(str, item.path)) or '<definition>'}: "
+            f"{item.message} {item.hint}"
+            for item in report.issues
+        ]
+        super().__init__("\n".join(lines) or "No validated definition")
+
+
+def check_definition(
+    definition: PluginDefinition | Mapping[str, Any],
+    *,
+    check_packaged_resources: bool = True,
+    load_entrypoints: bool = False,
+    external_services: Mapping[str, ServiceSpec] | None = None,
+    service_examples: Iterable[ServiceExample] = (),
+    resource_examples: Iterable[ResourceExample] = (),
+) -> ContractReport:
+    """Validate a definition and report repairable issues.
+
+    Entry-point imports are opt-in because importing plugin Python can execute
+    code. Packaged document checks operate in the current Python environment;
+    use a clean wheel install to verify distribution contents. MCP schemas may
+    narrow service schemas. Examples test concrete compatibility, never general
+    JSON Schema implication or Core admission/authorization.
+    """
+    issues: list[ContractIssue] = []
+
+    def issue(code, path, message, hint):
+        issues.append(ContractIssue(code, tuple(path), message, hint))
+
+    raw = (
+        definition.model_dump(mode="python")
+        if isinstance(definition, PluginDefinition)
+        else dict(definition)
+    )
+    try:
+        # Revalidate model instances too: SDK model_copy/model_construct can bypass
+        # validators. A JSON round trip also rejects non-JSON/NaN schema values.
+        validate({}, raw)
+        checked = PluginDefinition.model_validate(raw)
+    except ValidationError as error:
+        for detail in error.errors(include_input=False, include_url=False):
+            if not detail["loc"] and "undeclared scope" in detail["msg"]:
+                # The SDK has already rejected this cross-field contract. Add
+                # exact source locations to its root-level diagnostic rather
+                # than maintaining an alternative authorization validator.
+                known = set(raw.get("authorization", {}).get("scopes", {}))
+                known.update(SYSTEM_SCOPES)
+                locations = []
+                for service, spec in raw.get("services", {}).items():
+                    for method, method_spec in spec.get("methods", {}).items():
+                        locations.append(
+                            (
+                                (
+                                    "services",
+                                    service,
+                                    "methods",
+                                    method,
+                                    "required_scopes",
+                                ),
+                                method_spec.get("required_scopes", ()),
+                            )
+                        )
+                for index, resource in enumerate(raw.get("resources", ())):
+                    locations.append(
+                        (
+                            ("resources", index, "required_scopes"),
+                            resource.get("required_scopes", ()),
+                        )
+                    )
+                for path, scopes in locations:
+                    for index, scope in enumerate(scopes):
+                        if scope not in known:
+                            issue(
+                                "scope_undeclared",
+                                (*path, index),
+                                f"Scope {scope} is not declared in this definition.",
+                                "Declare an owned scope or use an SDK system scope.",
+                            )
+                if issues:
+                    continue
+            issue(
+                "definition_invalid",
+                detail["loc"],
+                detail["msg"],
+                "Repair this field using the public SDK PluginDefinition contract.",
+            )
+        return ContractReport(None, tuple(issues))
+    except (SchemaError, TypeError, ValueError) as error:
+        issue(
+            "definition_invalid",
+            (),
+            f"SDK schema/JSON validation failed ({type(error).__name__}).",
+            "Use valid Draft 2020-12 schemas and finite JSON values.",
+        )
+        return ContractReport(None, tuple(issues))
+
+    services = {**(external_services or {}), **checked.services}
+
+    def target(service, method, path):
+        spec = services.get(service)
+        result = spec.methods.get(method) if spec else None
+        if result is None:
+            issue(
+                "resource_target_missing",
+                path,
+                f"No declared method {service}/{method}.",
+                "Declare the target or supply its ServiceSpec in external_services.",
+            )
+        return result
+
+    def example(schema, value, path):
+        try:
+            validate(schema, value)
+        except SchemaValidationError as error:
+            issue(
+                "example_schema_invalid",
+                (*path, *error.absolute_path),
+                f"Example violates the schema's {error.validator} constraint.",
+                "Correct the example or schema; instance values are omitted.",
+            )
+        except (SchemaError, TypeError, ValueError) as error:
+            issue(
+                "example_schema_invalid",
+                path,
+                f"Example/schema is invalid JSON ({type(error).__name__}).",
+                "Use finite JSON values and valid Draft 2020-12 schemas.",
+            )
+
+    def entrypoint(reference, path):
+        try:
+            result = load_object(reference)
+            if not callable(result):
+                raise TypeError("Entry point must be callable")
+        except Exception as error:
+            issue(
+                "entrypoint_unavailable",
+                path,
+                f"Could not load callable {reference} ({type(error).__name__}).",
+                "Install the wheel and its dependencies; export a public callable.",
+            )
+
+    if load_entrypoints:
+        entrypoint(checked.factory, ("factory",))
+    mcp = {}
+    for index, resource in enumerate(checked.resources):
+        path = ("resources", index)
+        if resource.kind == "mcp.tool":
+            spec = target(
+                resource.data["service"],
+                resource.data["method"],
+                (*path, "data", "method"),
+            )
+            mcp[resource.id] = (resource, spec, path)
+            if resource.data["service"] not in checked.services and resource.data[
+                "service"
+            ] not in {requirement.service for requirement in checked.requires.values()}:
+                issue(
+                    "resource_dependency_undeclared",
+                    (*path, "data", "service"),
+                    "External resource target has no declared service requirement.",
+                    "Declare a Requirement for the external target service.",
+                )
+            elif spec and resource.data["service"] not in checked.services:
+                supplied_version = services[resource.data["service"]].version
+                for alias, requirement in checked.requires.items():
+                    if (
+                        requirement.service == resource.data["service"]
+                        and requirement.version != supplied_version
+                    ):
+                        issue(
+                            "resource_dependency_version_mismatch",
+                            ("requires", alias, "version"),
+                            f"Required service v{requirement.version}, "
+                            f"supplied ServiceSpec v{supplied_version}.",
+                            "Supply a matching service or revise the required version.",
+                        )
+            if spec and spec.audience != "business":
+                issue(
+                    "resource_target_not_business",
+                    (*path, "data", "method"),
+                    "MCP tools can expose only business service methods.",
+                    "Target a business method with explicit scopes.",
+                )
+        elif resource.kind == "dashboard.page" and load_entrypoints:
+            entrypoint(resource.data["entrypoint"], (*path, "data", "entrypoint"))
+        elif resource.kind == "llms.document" and check_packaged_resources:
+            try:
+                document = files(resource.data["package"]).joinpath(
+                    resource.data["path"]
+                )
+                if not document.is_file():
+                    raise FileNotFoundError()
+                content = document.read_text(encoding="utf-8")
+                if len(content.encode("utf-8")) > 256 * 1024:
+                    issue(
+                        "packaged_document_too_large",
+                        (*path, "data", "path"),
+                        "Document exceeds the SDK worker's 256 KiB limit.",
+                        "Split or shorten the packaged document.",
+                    )
+            except Exception as error:
+                issue(
+                    "packaged_document_unavailable",
+                    (*path, "data", "path"),
+                    f"Packaged UTF-8 document is unavailable ({type(error).__name__}).",
+                    "Include the document in the wheel; verify a clean installation.",
+                )
+
+    for index, item in enumerate(service_examples):
+        path = ("service_examples", index)
+        spec = target(item.service, item.method, (*path, "method"))
+        if spec:
+            example(spec.input_schema, item.input, (*path, "input"))
+            if item.output is not _ABSENT:
+                example(spec.output_schema, item.output, (*path, "output"))
+    for index, item in enumerate(resource_examples):
+        path = ("resource_examples", index)
+        found = mcp.get(item.resource_id)
+        if found is None:
+            issue(
+                "resource_example_missing",
+                (*path, "resource_id"),
+                f"No MCP resource {item.resource_id}.",
+                "Use a declared MCP resource ID.",
+            )
+            continue
+        resource, spec, _ = found
+        example(resource.data.get("input_schema", {}), item.input, (*path, "input"))
+        if spec:
+            example(spec.input_schema, item.input, (*path, "service_input"))
+        if item.output is not _ABSENT:
+            example(
+                resource.data.get("output_schema", {}), item.output, (*path, "output")
+            )
+            if spec:
+                example(spec.output_schema, item.output, (*path, "service_output"))
+    return ContractReport(checked, tuple(issues))

toolframe_testkit/harness.py

@@ -0,0 +1,566 @@
+"""An owned temporary application driven only by installed CLI and HTTP APIs."""
+
+from __future__ import annotations
+
+import json
+import math
+import os
+import re
+import shutil
+import socket
+import subprocess
+import tempfile
+import threading
+import time
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Literal
+from urllib.parse import quote, urlsplit
+
+import httpx
+import psutil
+import tomlkit
+
+
+class HarnessError(RuntimeError):
+    """A CLI, operation, readiness or cleanup failure; output remains on disk."""
+
+
+class HarnessTimeout(HarnessError):
+    """A bounded application/command deadline expired."""
+
+
+@dataclass(frozen=True)
+class CLIResult:
+    returncode: int
+    stdout: str
+    stderr: str
+    output_file: Path
+
+    def json(self) -> Any:
+        return json.loads(self.stdout)
+
+
+def _positive(value: float, name: str) -> float:
+    if not math.isfinite(value) or value <= 0:
+        raise ValueError(f"{name} must be finite and positive")
+    return value
+
+
+def _find_link(value: str | Path) -> str:
+    value = str(value)
+    parsed = urlsplit(value)
+    if parsed.scheme in {"https", "file"}:
+        if parsed.username or parsed.password or parsed.query or parsed.fragment:
+            raise ValueError(
+                "find_links URLs must not contain credentials/query/fragment"
+            )
+        return value
+    if parsed.scheme:
+        raise ValueError("find_links must be local paths or https/file URLs")
+    return str(Path(value).expanduser().resolve())
+
+
+class _ProcessWatch:
+    """Retain identities of owned descendants even if their parent later exits."""
+
+    def __init__(self, process: subprocess.Popen):
+        self.process = psutil.Process(process.pid)
+        self._children: dict[tuple[int, float], psutil.Process] = {}
+        self._lock = threading.Lock()
+        self._done = threading.Event()
+        self._thread = threading.Thread(target=self._run, daemon=True)
+        self._thread.start()
+
+    def snapshot(self) -> list[psutil.Process]:
+        try:
+            children = self.process.children(recursive=True)
+            with self._lock:
+                for child in children:
+                    try:
+                        self._children[(child.pid, child.create_time())] = child
+                    except psutil.NoSuchProcess:
+                        pass
+        except psutil.NoSuchProcess:
+            pass
+        with self._lock:
+            return list(self._children.values())
+
+    def _run(self):
+        while not self._done.is_set():
+            self.snapshot()
+            self._done.wait(0.05)
+
+    def stop(self) -> list[psutil.Process]:
+        self._done.set()
+        self._thread.join(timeout=2)
+        return self.snapshot()
+
+
+class ApplicationHarness:
+    """Start a private application with an explicitly supplied installed command.
+
+    The application home is always newly allocated and removed on close. Logs
+    survive under a unique directory in output_directory. Put that directory in
+    an ignored build location. No existing application home can be supplied.
+    local_control=False makes loopback visitors use Core's normal admission.
+    This harness does not simulate principals or bypass admission checks.
+    """
+
+    def __init__(
+        self,
+        core_command: str | Path | Sequence[str],
+        *,
+        output_directory: Path,
+        port: int | None = None,
+        find_links: Sequence[str | Path] = (),
+        wheelhouse: Path | None = None,
+        uv_command: str | Path | None = None,
+        startup_timeout: float = 60,
+        command_timeout: float = 120,
+        shutdown_timeout: float = 45,
+        poll_interval: float = 0.1,
+        local_control: bool = False,
+    ):
+        command = (
+            [str(core_command)]
+            if isinstance(core_command, (str, Path))
+            else list(core_command)
+        )
+        executable = shutil.which(command[0]) if command else None
+        if not executable:
+            raise ValueError("Supply an available installed Core command or binary")
+        command[0] = str(Path(executable).resolve())
+        if port is not None and not 1 <= port <= 65535:
+            raise ValueError("port must be between 1 and 65535")
+        self.startup_timeout = _positive(startup_timeout, "startup_timeout")
+        self.command_timeout = _positive(command_timeout, "command_timeout")
+        self.shutdown_timeout = _positive(shutdown_timeout, "shutdown_timeout")
+        self.poll_interval = _positive(poll_interval, "poll_interval")
+        self.core_command = tuple(command)
+        self.find_links = tuple(_find_link(link) for link in find_links)
+        self.local_control = local_control
+        if wheelhouse is not None:
+            wheelhouse = wheelhouse.expanduser().resolve()
+            if not wheelhouse.is_dir():
+                raise ValueError("wheelhouse must be an existing directory")
+            self.find_links += (str(wheelhouse),)
+        if uv_command is not None and not shutil.which(str(uv_command)):
+            raise ValueError("uv_command must be an available executable")
+
+        root = Path(output_directory).expanduser().resolve()
+        root.mkdir(parents=True, exist_ok=True)
+        self.output_directory = Path(tempfile.mkdtemp(prefix="toolframe-", dir=root))
+        self._temporary = tempfile.TemporaryDirectory(prefix="toolframe-testkit-")
+        self._root = Path(self._temporary.name).resolve()
+        self._home = self._root / "application"
+        self.home.mkdir(mode=0o700)
+        if port is None:
+            with socket.socket() as reservation:
+                reservation.bind(("127.0.0.1", 0))
+                port = reservation.getsockname()[1]
+        self.port = port
+        self.base_url = f"http://127.0.0.1:{port}"
+        # Retain only platform essentials. Global Toolframe/Python/uv/DB settings,
+        # proxy credentials and user config are never inherited by the child.
+        self._environment = {
+            name: os.environ[name]
+            for name in ("PATH", "SYSTEMROOT", "WINDIR", "LANG", "LC_ALL")
+            if name in os.environ
+        }
+        private_home = self._root / "user"
+        private_home.mkdir(mode=0o700)
+        temporary = self._root / "tmp"
+        temporary.mkdir(mode=0o700)
+        self._environment.update(
+            HOME=str(private_home),
+            USERPROFILE=str(private_home),
+            XDG_CONFIG_HOME=str(private_home / ".config"),
+            XDG_CACHE_HOME=str(private_home / ".cache"),
+            TMPDIR=str(temporary),
+            TMP=str(temporary),
+            TEMP=str(temporary),
+            UV_CACHE_DIR=str(self._root / "uv-cache"),
+            UV_NO_CONFIG="1",
+            TOOLFRAME_RUNTIME_ROOT=str(self._root / "launcher-runtime"),
+            PYTHONUNBUFFERED="1",
+        )
+        if uv_command is not None:
+            self._environment["TOOLFRAME_UV"] = str(Path(shutil.which(str(uv_command))))
+        if wheelhouse is not None:
+            self._environment["TOOLFRAME_WHEELHOUSE"] = str(wheelhouse)
+        self._client = httpx.Client(base_url=self.base_url, timeout=5, trust_env=False)
+        self._process: subprocess.Popen | None = None
+        self._serve_watch: _ProcessWatch | None = None
+        self._reader_threads: list[threading.Thread] = []
+        self._initialized = False
+        self._closed = False
+        self._log_counter = 0
+
+    @property
+    def home(self) -> Path:
+        return self._home
+
+    def __enter__(self) -> ApplicationHarness:
+        try:
+            return self.start()
+        except BaseException:
+            self.close()
+            raise
+
+    def __exit__(self, *_):
+        self.close()
+
+    def _ensure_open(self):
+        if self._closed:
+            raise HarnessError("ApplicationHarness has been closed")
+
+    def _token(self) -> str:
+        path = self.home / ".toolframe" / "management.token"
+        if path.is_symlink() or not path.resolve().is_relative_to(self.home):
+            raise HarnessError("Management token must stay inside the owned home")
+        try:
+            token = path.read_text(encoding="utf-8").strip()
+        except OSError:
+            raise HarnessError(
+                "Owned application's management token is unavailable"
+            ) from None
+        if not token:
+            raise HarnessError("Owned application's management token is empty")
+        return token
+
+    def _redact(self, text: str) -> str:
+        try:
+            text = text.replace(self._token(), "[REDACTED]")
+        except HarnessError:
+            pass
+        text = re.sub(r"(?i)(Bearer\s+)[^\s\"'<>]+", r"\1[REDACTED]", text)
+        return re.sub(r"(?i)([?&]token=)[^&\s\"'<>]+", r"\1[REDACTED]", text)
+
+    def _argv(self, args: Sequence[str]) -> list[str]:
+        return [*self.core_command, "--home", str(self.home), "--json", *args]
+
+    def _log(self, label: str) -> Path:
+        self._log_counter += 1
+        return self.output_directory / f"{self._log_counter:03d}-{label}.log"
+
+    @staticmethod
+    def _cleanup_descendants(children: list[psutil.Process]):
+        # psutil Process identity includes creation time, preventing signals from
+        # being sent to a reused PID. Only observed descendants of our child count.
+        alive = [child for child in children if child.is_running()]
+        for child in reversed(alive):
+            try:
+                child.terminate()
+            except psutil.NoSuchProcess:
+                pass
+        _, alive = psutil.wait_procs(alive, timeout=2)
+        for child in alive:
+            try:
+                child.kill()
+            except psutil.NoSuchProcess:
+                pass
+        _, alive = psutil.wait_procs(alive, timeout=2)
+        if alive:
+            raise HarnessError("Owned child processes did not terminate")
+
+    def _terminate(self, process: subprocess.Popen, watch: _ProcessWatch):
+        children = watch.snapshot()
+        try:
+            if process.poll() is None:
+                process.terminate()
+                try:
+                    process.wait(timeout=self.shutdown_timeout)
+                except subprocess.TimeoutExpired:
+                    # Recheck for children spawned during graceful shutdown.
+                    children += watch.snapshot()
+                    process.kill()
+                    process.wait(timeout=5)
+        finally:
+            children += watch.stop()
+            self._cleanup_descendants(list(set(children)))
+
+    def cli(
+        self, *args: str, timeout: float | None = None, check: bool = True
+    ) -> CLIResult:
+        """Run a CLI action with a deadline and sanitized retained output.
+
+        Avoid credential-producing actions: their output is intentionally redacted.
+        Args are passed as an argv list; no shell is invoked.
+        """
+        self._ensure_open()
+        if any(arg == "--home" or arg.startswith("--home=") for arg in args):
+            raise ValueError(
+                "CLI home cannot be overridden outside the owned application"
+            )
+        timeout = _positive(
+            self.command_timeout if timeout is None else timeout, "timeout"
+        )
+        output = self._log("cli")
+        process = subprocess.Popen(
+            self._argv(args),
+            cwd=self.home,
+            env=self._environment,
+            stdin=subprocess.DEVNULL,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+        )
+        watch = _ProcessWatch(process)
+        expired = False
+        try:
+            stdout, stderr = process.communicate(timeout=timeout)
+        except subprocess.TimeoutExpired:
+            expired = True
+            self._terminate(process, watch)
+            stdout, stderr = process.communicate(timeout=5)
+        else:
+            self._terminate(process, watch)
+        clean_stdout = self._redact(stdout.decode("utf-8", errors="replace"))
+        clean_stderr = self._redact(stderr.decode("utf-8", errors="replace"))
+        output.write_text(
+            clean_stdout + "\n[stderr]\n" + clean_stderr, encoding="utf-8"
+        )
+        result = CLIResult(process.returncode, clean_stdout, clean_stderr, output)
+        if expired:
+            raise HarnessTimeout(f"CLI deadline expired; output: {output}")
+        if check and result.returncode:
+            raise HarnessError(f"CLI exited {result.returncode}; output: {output}")
+        return result
+
+    def initialize(self) -> ApplicationHarness:
+        self._ensure_open()
+        if self._initialized:
+            return self
+        self.cli("init", "--host", "127.0.0.1", "--port", str(self.port))
+        # This is the public composition file in our newly created home. No
+        # existing/user application composition is accepted by this harness.
+        path = self.home / "toolframe.toml"
+        document = tomlkit.parse(path.read_text(encoding="utf-8"))
+        document.setdefault("packages", {})["find_links"] = list(self.find_links)
+        document.setdefault("access", {})["local_control"] = self.local_control
+        path.write_text(tomlkit.dumps(document), encoding="utf-8")
+        self._initialized = True
+        return self
+
+    def _capture(self, stream, output: Path):
+        with output.open("w", encoding="utf-8") as handle:
+            for line in iter(stream.readline, b""):
+                handle.write(self._redact(line.decode("utf-8", errors="replace")))
+                handle.flush()
+        stream.close()
+
+    def start(self) -> ApplicationHarness:
+        self._ensure_open()
+        if self._process is not None:
+            raise HarnessError("Core is already started; call stop() before restarting")
+        self.initialize()
+        self._process = subprocess.Popen(
+            self._argv(("serve",)),
+            cwd=self.home,
+            env=self._environment,
+            stdin=subprocess.DEVNULL,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+        )
+        self._serve_watch = _ProcessWatch(self._process)
+        for label, stream in (
+            ("serve-stdout", self._process.stdout),
+            ("serve-stderr", self._process.stderr),
+        ):
+            reader = threading.Thread(
+                target=self._capture, args=(stream, self._log(label)), daemon=True
+            )
+            reader.start()
+            self._reader_threads.append(reader)
+        deadline = time.monotonic() + self.startup_timeout
+        try:
+            while time.monotonic() < deadline:
+                self._alive()
+                try:
+                    response = self.request(
+                        "GET",
+                        "/_toolframe/v1/readiness",
+                        management=True,
+                        timeout=min(5, max(0.01, deadline - time.monotonic())),
+                    )
+                    if response.status_code == 200 and response.json().get("ready"):
+                        return self
+                except httpx.HTTPError, HarnessError:
+                    pass
+                time.sleep(min(self.poll_interval, max(0, deadline - time.monotonic())))
+            raise HarnessTimeout(
+                f"Core readiness deadline expired; output: {self.output_directory}"
+            )
+        except BaseException:
+            self.stop()
+            raise
+
+    def _alive(self):
+        if self._process is None or self._process.poll() is not None:
+            raise HarnessError(f"Core is not running; output: {self.output_directory}")
+
+    def stop(self):
+        """Gracefully stop this Core, then reap only its observed descendants."""
+        if self._process is not None:
+            try:
+                self._terminate(self._process, self._serve_watch)
+            finally:
+                self._process = None
+                self._serve_watch = None
+                for reader in self._reader_threads:
+                    reader.join(timeout=5)
+                self._reader_threads.clear()
+
+    def close(self):
+        if self._closed:
+            return
+        try:
+            self.stop()
+        finally:
+            self._client.close()
+            self._temporary.cleanup()
+            self._closed = True
+
+    def request(
+        self, method: str, path: str, *, management: bool = False, **kwargs
+    ) -> httpx.Response:
+        """Send an HTTP request to this application, optionally as its operator."""
+        self._ensure_open()
+        if not path.startswith("/") or path.startswith("//") or urlsplit(path).netloc:
+            raise ValueError("Request path must be relative to this application")
+        headers = httpx.Headers(kwargs.pop("headers", {}))
+        if management:
+            headers["authorization"] = f"Bearer {self._token()}"
+        # Explicitly forbid following redirects which might transmit the token
+        # to an external target; consumers may inspect Location themselves.
+        if kwargs.pop("follow_redirects", False):
+            raise ValueError("Harness requests must not follow redirects")
+        return self._client.request(method, path, headers=headers, **kwargs)
+
+    def status(self) -> dict:
+        response = self.request("GET", "/_toolframe/v1/status", management=True)
+        response.raise_for_status()
+        return response.json()
+
+    def _poll_response(self, path: str, deadline: float) -> dict | None:
+        try:
+            response = self.request(
+                "GET",
+                path,
+                management=True,
+                timeout=min(5, max(0.01, deadline - time.monotonic())),
+            )
+        except httpx.TransportError:
+            return None
+        if response.status_code >= 500 or response.status_code == 429:
+            return None
+        response.raise_for_status()
+        return response.json()
+
+    def wait_for_plugin(
+        self, plugin_id: str, *, healthy: bool = True, timeout: float | None = None
+    ) -> dict:
+        deadline = time.monotonic() + _positive(
+            self.command_timeout if timeout is None else timeout, "timeout"
+        )
+        while time.monotonic() < deadline:
+            self._alive()
+            report = self._poll_response("/_toolframe/v1/status", deadline)
+            for row in (report or {}).get("plugins", []):
+                if row["id"] == plugin_id and bool(row.get("healthy")) == healthy:
+                    return row
+            time.sleep(min(self.poll_interval, max(0, deadline - time.monotonic())))
+        raise HarnessTimeout(
+            f"Plugin health deadline expired; output: {self.output_directory}"
+        )
+
+    def install_plugin(
+        self, source: str | Path, *, plugin_id: str | None = None, enable: bool = True
+    ) -> CLIResult:
+        if isinstance(source, Path):
+            normalized = str(source.expanduser().resolve())
+        else:
+            prefix, _, value = source.partition(":")
+            if prefix in {"wheel", "path"}:
+                normalized = prefix + ":" + str(Path(value).expanduser().resolve())
+            elif source.startswith(("pypi:", "git+", "https://")):
+                normalized = source
+            else:
+                normalized = str(Path(source).expanduser().resolve())
+        args = ["plugin", "add", normalized]
+        if plugin_id is not None:
+            args += ["--id", plugin_id]
+        if enable:
+            args.append("--enable")
+        return self.cli(*args)
+
+    def plugin_action(self, action: str, plugin_id: str, *args: str) -> CLIResult:
+        if action not in {
+            "validate",
+            "update",
+            "reload",
+            "enable",
+            "disable",
+            "remove",
+            "stop",
+            "status",
+            "logs",
+        }:
+            raise ValueError("Unknown public plugin action")
+        return self.cli("plugin", action, plugin_id, *args)
+
+    def operation(
+        self,
+        action: str,
+        params: Mapping[str, Any] | None = None,
+        *,
+        timeout: float | None = None,
+    ) -> dict:
+        """Submit a public management operation and await its terminal result."""
+        deadline = time.monotonic() + _positive(
+            self.command_timeout if timeout is None else timeout, "timeout"
+        )
+        response = self.request(
+            "POST",
+            "/_toolframe/v1/operations",
+            management=True,
+            json={"action": action, "params": dict(params or {})},
+            timeout=min(5, max(0.01, deadline - time.monotonic())),
+        )
+        response.raise_for_status()
+        identifier = response.json()["id"]
+        while time.monotonic() < deadline:
+            self._alive()
+            result = self._poll_response(
+                f"/_toolframe/v1/operations/{quote(identifier, safe='')}", deadline
+            )
+            if result and result["status"] not in {"queued", "running", "cancelling"}:
+                if result["status"] != "succeeded":
+                    raise HarnessError(
+                        f"Operation {identifier} {result['status']}; "
+                        f"output: {self.output_directory}"
+                    )
+                return result
+            time.sleep(min(self.poll_interval, max(0, deadline - time.monotonic())))
+        raise HarnessTimeout(
+            f"Operation deadline expired; output: {self.output_directory}"
+        )
+
+    def service_call(
+        self,
+        owner: str,
+        service: str,
+        method: str,
+        params: Any,
+        *,
+        entrance: Literal["visitor", "public"] = "visitor",
+        management: bool = False,
+    ) -> httpx.Response:
+        if entrance not in {"visitor", "public"}:
+            raise ValueError("entrance must be visitor or public")
+        prefix = "/api" if entrance == "public" else "/_toolframe/v1/services"
+        path = "/".join(quote(value, safe="") for value in (owner, service, method))
+        return self.request(
+            "POST", f"{prefix}/{path}", management=management, json=params
+        )

toolframe_testkit/pytest_plugin.py

@@ -0,0 +1,79 @@
+"""Opt-in pytest integration: load with pytest_plugins or pytest -p.
+
+There is intentionally no automatic pytest11 entry point, and importing the
+runtime package does not import pytest or start an application.
+"""
+
+from __future__ import annotations
+
+import shlex
+from contextlib import ExitStack
+from pathlib import Path
+
+import pytest
+
+from .contracts import check_definition
+from .harness import ApplicationHarness
+
+
+def pytest_addoption(parser):
+    group = parser.getgroup("toolframe")
+    group.addoption(
+        "--toolframe-core-command",
+        help="Explicit installed Core argv (shell-style quoting; no shell executes it)",
+    )
+    group.addoption(
+        "--toolframe-find-links",
+        action="append",
+        default=[],
+        help="Wheel directory or https/file URL (repeatable)",
+    )
+    group.addoption(
+        "--toolframe-output-directory",
+        default="build/toolframe-testkit",
+        help="Ignored directory retaining sanitized child output",
+    )
+
+
+@pytest.fixture
+def toolframe_definition_checker():
+    """The public definition checker, convenient for parametrized plugin tests."""
+    return check_definition
+
+
+@pytest.fixture
+def toolframe_harness_factory(request):
+    """Create owned harnesses and close all of them when the test ends.
+
+    A supplied command option is required only when this factory is called.
+    Explicit keyword arguments override CLI configuration.
+    """
+    cleanup = ExitStack()
+
+    def factory(**kwargs):
+        command = kwargs.pop("core_command", None)
+        if command is None:
+            configured = request.config.getoption("--toolframe-core-command")
+            if not configured:
+                pytest.skip("Supply --toolframe-core-command for installed Core tests")
+            command = shlex.split(configured)
+        kwargs.setdefault(
+            "output_directory",
+            Path(request.config.getoption("--toolframe-output-directory")),
+        )
+        kwargs.setdefault(
+            "find_links", request.config.getoption("--toolframe-find-links")
+        )
+        app = ApplicationHarness(command, **kwargs)
+        cleanup.callback(app.close)
+        return app
+
+    with cleanup:
+        yield factory
+
+
+@pytest.fixture
+def toolframe_application(toolframe_harness_factory):
+    """A started private application; skip when no installed command is supplied."""
+    with toolframe_harness_factory() as app:
+        yield app

toolframe_testkit-0.1.2.dist-info/METADATA

@@ -0,0 +1,108 @@
+Metadata-Version: 2.5
+Name: toolframe-testkit
+Version: 0.1.2
+Summary: Public contract checks and installed-application tests for Toolframe developers
+Project-URL: Homepage, https://github.com/toolframe/toolframe-testkit
+Project-URL: Repository, https://github.com/toolframe/toolframe-testkit
+Project-URL: Documentation, https://github.com/toolframe/toolframe-testkit#readme
+Project-URL: Issues, https://github.com/toolframe/toolframe-testkit/issues
+Author-email: byfb <62798989+byfb@users.noreply.github.com>
+License-Expression: Apache-2.0
+License-File: LICENSE
+Requires-Python: >=3.14
+Requires-Dist: httpx<1,>=0.28
+Requires-Dist: jsonschema<5,>=4.23
+Requires-Dist: psutil<9,>=7
+Requires-Dist: pydantic<3,>=2.12.5
+Requires-Dist: tomlkit<1,>=0.13
+Requires-Dist: toolframe-sdk==0.3.0
+Provides-Extra: pytest
+Requires-Dist: pytest<10,>=8; extra == 'pytest'
+Description-Content-Type: text/markdown
+
+# Toolframe testkit
+
+Toolframe testkit is a Python library for checking a plugin's public SDK contract
+and exercising an installed Toolframe Core application through its public CLI and
+HTTP APIs. It is a developer test dependency, not a Core runtime plugin. Version
+0.1.2 supports Python 3.14 or newer and pins `toolframe-sdk==0.3.0`.
+
+The package is Apache-2.0 licensed. It does not install Core, choose a global
+application, or load a pytest plugin automatically.
+
+## Quick start
+
+Install the checker and opt-in pytest integration from PyPI:
+
+```sh
+python -m pip install 'toolframe-testkit[pytest]==0.1.2'
+```
+
+The SDK is installed as a dependency. See
+[development and verification](docs/development.md) for source builds and local
+wheelhouse checks.
+
+Check a public SDK definition in a unit test:
+
+```python
+from toolframe_testkit import check_definition
+from my_plugin.definition import plugin
+
+
+def test_plugin_contract():
+    check_definition(plugin).assert_valid()
+```
+
+For installed Core tests, explicitly load the pytest plugin in `conftest.py`:
+
+```python
+pytest_plugins = ["toolframe_testkit.pytest_plugin"]
+```
+
+Then supply an installed Core command when running the test. The plugin creates a
+new private application home, starts Core on a loopback port, and removes the home
+after the test. A minimal `tests/test_application.py` can check readiness without
+installing a plugin:
+
+```python
+def test_application(toolframe_application):
+    assert toolframe_application.status()["ready"] is True
+```
+
+Child output is retained below the selected ignored build directory:
+
+```sh
+pytest tests/test_application.py \
+  --toolframe-core-command "$(command -v toolframe-core)" \
+  --toolframe-output-directory build/installed-core
+```
+
+See [contract checks](docs/contracts.md) for definition and example validation,
+[the application harness](docs/harness.md) for installed Core testing, and
+[development and verification](docs/development.md) for wheel builds and the
+real-Core fixture run. The complete standalone scenario is in
+[examples/installed_application.py](examples/installed_application.py).
+
+## Support limits
+
+The harness is deliberately explicit: callers choose the installed Core command,
+wheel links, and output directory. It uses public Core CLI and HTTP endpoints and
+does not import Core internals or edit an existing user application. The default
+`local_control=False` exercises Core's normal admission checks; setting it to
+`True` is available only when a test specifically needs local control behavior.
+
+The pytest integration is opt-in through `pytest_plugins` or `pytest -p`; there is
+no `pytest11` auto-entry point. `check_definition` validates the SDK model, target
+references, resource scopes, packaged UTF-8 documents, and supplied concrete
+examples. Example checks do not prove general JSON Schema implication or Core
+authorization behavior. MCP resource schemas may intentionally narrow a service
+schema.
+
+The installed-Core harness covers public readiness, status, plugin lifecycle,
+service admission, authorization, schema rejection, and retained data. Browser
+interaction, provider-specific protocols, and external service provisioning need
+separate acceptance checks. The harness runs local processes; it does not manage
+remote deployments. It redacts its owned management token and common bearer/token
+query forms, but cannot guarantee redaction of arbitrary plugin output.
+
+For release validation and Trusted Publisher setup, see [releasing](docs/releasing.md).

toolframe_testkit-0.1.2.dist-info/RECORD

@@ -0,0 +1,9 @@
+toolframe_testkit/__init__.py,sha256=bg7_duMcy9ie52gjyFPcBfdm1TT5GPJcNVDfsHhZFjw,570
+toolframe_testkit/contracts.py,sha256=EaitfEi5p8EmbAshYjzs-257iyWl8WWU3BcKB5SuyUA,12072
+toolframe_testkit/harness.py,sha256=osCzovKKJdup2QS84-sm1SyyIWE5qioLHRS2OgTfJ48,21965
+toolframe_testkit/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+toolframe_testkit/pytest_plugin.py,sha256=mK3NqncKAkK1UtDoZJ0s0bTMxr1TWmxq7I7fbx5v2Sw,2431
+toolframe_testkit-0.1.2.dist-info/METADATA,sha256=Ll9jOMVeYPyZHgBRvB7gB50v65yfG3kZ6oc8ukHPSAA,4396
+toolframe_testkit-0.1.2.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+toolframe_testkit-0.1.2.dist-info/licenses/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
+toolframe_testkit-0.1.2.dist-info/RECORD,,

toolframe_testkit-0.1.2.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.4
+Root-Is-Purelib: true
+Tag: py3-none-any

toolframe_testkit-0.1.2.dist-info/licenses/LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.