toolframe-core 0.6.2__py3-none-any.whl

toolframe_core/process.py

@@ -0,0 +1,212 @@
+"""Own a worker process, its protocol channel, stderr tail and actual exit."""
+
+from __future__ import annotations
+
+import asyncio
+import contextlib
+import os
+import signal
+from collections import deque
+
+from toolframe_sdk import MAX_FRAME_BYTES, PluginError
+from toolframe_sdk.rpc import RpcPeer
+
+
+def child_environment():
+    """Build/preflight environments exclude Core and telemetry credentials."""
+    return {
+        key: value
+        for key, value in os.environ.items()
+        if key not in ("PYTHONPATH", "PYTHONHOME")
+        and not key.startswith("TOOLFRAME_CORE_")
+        and not key.startswith("OTEL_")
+        and key not in {"TOOLFRAME_TRACING_ENABLED", "TOOLFRAME_LOGS_LEVEL"}
+    }
+
+
+class WorkerProcess:
+    def __init__(
+        self,
+        process,
+        handler,
+        *,
+        log_callback=None,
+        tracing=None,
+        trace_attributes=None,
+    ):
+        self.process = process
+        self.expected_exit = False
+        self.logs = deque(maxlen=200)
+        self.log_callback = log_callback
+        self.trace_attributes = trace_attributes or {}
+        self.peer = RpcPeer(
+            process.stdout,
+            process.stdin,
+            handler,
+            tracing=tracing,
+            trace_attributes={
+                "toolframe.target.plugin.id": self.trace_attributes.get(
+                    "toolframe.plugin.id", ""
+                ),
+                "toolframe.target.generation": self.trace_attributes.get(
+                    "toolframe.plugin.generation", ""
+                ),
+            },
+        )
+        self.reader_task = asyncio.create_task(self.peer.run())
+        self.log_task = asyncio.create_task(self._logs())
+
+    @classmethod
+    async def start(
+        cls,
+        python,
+        entrypoint,
+        handler,
+        *,
+        cwd,
+        log_callback=None,
+        host_token=None,
+        tracing=None,
+        trace_attributes=None,
+    ):
+        environment = child_environment()
+        if tracing:
+            environment.update(
+                tracing.child_environment(
+                    (trace_attributes or {}).get("service.name", "toolframe.worker")
+                )
+            )
+        process = await asyncio.create_subprocess_exec(
+            str(python),
+            "-I",
+            "-u",
+            "-m",
+            "toolframe_sdk.worker",
+            "--entrypoint",
+            entrypoint,
+            *(["--host-token", host_token] if host_token else []),
+            cwd=cwd,
+            env=environment,
+            stdin=asyncio.subprocess.PIPE,
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE,
+            limit=MAX_FRAME_BYTES,
+            start_new_session=True,
+        )
+        return cls(
+            process,
+            handler,
+            log_callback=log_callback,
+            tracing=tracing,
+            trace_attributes=trace_attributes,
+        )
+
+    @property
+    def pid(self):
+        return self.process.pid
+
+    async def _logs(self):
+        remainder = b""
+        while data := await self.process.stderr.read(4096):
+            remainder += data
+            lines = remainder.split(b"\n")
+            remainder = lines.pop()[-4096:]
+            for line in lines:
+                text = line[-4096:].decode(errors="replace")
+                self.logs.append(text)
+                if self.log_callback:
+                    self.log_callback(text)
+
+    async def request(self, method, params=None, *, timeout=30):
+        if method == "runtime.describe":
+            params = {**(params or {}), "logging": self.trace_attributes}
+            if self.peer.tracing:
+                params["tracing"] = self.trace_attributes
+        return await self.peer.request(method, params, timeout=timeout)
+
+    async def close(self, timeout=10, *, activated=True, allow_exited=False):
+        # A crashed worker may be replaced only after its entire process group
+        # is gone. A worker that fails during this disposal still fails cleanup.
+        exited_before_disposal = self.process.returncode is not None
+        self.expected_exit = True
+        if self.process.returncode is None:
+            if activated:
+                await self.request("runtime.dispose", timeout=timeout)
+            self.process.stdin.close()
+            try:
+                await asyncio.wait_for(self.process.wait(), timeout)
+            except TimeoutError as error:
+                raise PluginError(
+                    "cleanup_failed",
+                    "Worker did not exit after disposal",
+                    {"pid": self.pid},
+                ) from error
+        await self.peer.close()
+        if group_exists(self.pid):
+            raise PluginError(
+                "cleanup_failed",
+                "Worker process group still has descendants",
+                {"pgid": self.pid},
+            )
+        await asyncio.wait_for(
+            asyncio.gather(self.reader_task, self.log_task, return_exceptions=True),
+            timeout,
+        )
+        if self.process.returncode not in (0, None) and not (
+            allow_exited and exited_before_disposal
+        ):
+            raise PluginError(
+                "cleanup_failed",
+                "Worker exited unsuccessfully",
+                {"returncode": self.process.returncode},
+            )
+
+    async def force_stop(self):
+        self.expected_exit = True
+        await stop_group(self.pid)
+        if self.process.returncode is None:
+            await asyncio.wait_for(self.process.wait(), 5)
+        await self.peer.close()
+        await asyncio.wait_for(
+            asyncio.gather(self.reader_task, self.log_task, return_exceptions=True), 5
+        )
+
+
+def group_exists(pid):
+    try:
+        os.killpg(pid, 0)
+        return True
+    except ProcessLookupError:
+        return False
+    except PermissionError:
+        # EPERM does not prove absence. Darwin can also return it while the
+        # group's last descendants are zombies awaiting collection.
+        return True
+
+
+async def stop_group(pid):
+    for sig in (signal.SIGTERM, signal.SIGKILL):
+        if group_exists(pid):
+            with contextlib.suppress(ProcessLookupError, PermissionError):
+                os.killpg(pid, sig)
+            deadline = asyncio.get_running_loop().time() + 2
+            while group_exists(pid) and asyncio.get_running_loop().time() < deadline:
+                await asyncio.sleep(0.05)
+    if group_exists(pid):
+        raise PluginError(
+            "cleanup_failed",
+            "Process group did not exit after forced stop",
+            {"pgid": pid},
+        )
+
+
+async def process_inventory():
+    process = await asyncio.create_subprocess_exec(
+        "ps", "-axo", "pid=,pgid=,command=", stdout=asyncio.subprocess.PIPE
+    )
+    output, _ = await process.communicate()
+    return {
+        int(parts[0]): {"pgid": int(parts[1]), "command": parts[2]}
+        for line in output.decode().splitlines()
+        if len(parts := line.split(None, 2)) == 3
+    }

toolframe_core/redaction.py

@@ -0,0 +1,120 @@
+"""Redact management views without changing the configuration used by workers."""
+
+import re
+from urllib.parse import quote
+
+import tomlkit
+
+REDACTED = "[REDACTED]"
+SENSITIVE_KEY = re.compile(
+    r"password|passphrase|secret|token|api[_-]?key|authorization|credential",
+    re.IGNORECASE,
+)
+URL_AUTH = re.compile(r"((?:git\+)?(?:https?|ssh)://)[^\s/@]+@", re.IGNORECASE)
+URL_QUERY = re.compile(
+    r"([?&](?:token|api[_-]?key|password|secret|authorization|code|state|code_verifier)=)[^&\s]+",
+    re.IGNORECASE,
+)
+BEARER = re.compile(r"(\bBearer\s+)[^\s\"']+", re.IGNORECASE)
+MCP_KEY = re.compile(r"tf_(?:mcp|oauth|refresh|user)_[A-Za-z0-9_-]{43}")
+
+
+def secret_values(value, schema=None, *, key="", root=None):
+    schema = schema if isinstance(schema, dict) else {}
+    root = schema if root is None else root
+    reference = schema.get("$ref", "")
+    if reference.startswith("#/"):
+        resolved = root
+        for part in reference[2:].split("/"):
+            resolved = (
+                resolved.get(part.replace("~1", "/").replace("~0", "~"), {})
+                if isinstance(resolved, dict)
+                else {}
+            )
+        schema = {**(resolved if isinstance(resolved, dict) else {}), **schema}
+    protected = (
+        SENSITIVE_KEY.search(key)
+        or schema.get("writeOnly")
+        or schema.get("x-secret")
+        or schema.get("format") == "password"
+    )
+    if protected:
+        if isinstance(value, str) and value:
+            return {value}
+        if type(value) in (int, float):
+            return {str(value)}
+    result = set()
+    if isinstance(value, dict):
+        for name, item in value.items():
+            child = schema.get("properties", {}).get(name, {})
+            if protected:
+                child = {
+                    **(child if isinstance(child, dict) else {}),
+                    "writeOnly": True,
+                }
+            result.update(secret_values(item, child, key=name, root=root))
+    elif isinstance(value, (list, tuple)):
+        for item in value:
+            child = schema.get("items", {})
+            if protected:
+                child = {
+                    **(child if isinstance(child, dict) else {}),
+                    "writeOnly": True,
+                }
+            result.update(secret_values(item, child, key=key, root=root))
+    return result
+
+
+def redact(value, secrets, *, field=""):
+    if isinstance(value, dict):
+        if field == "request" and "config" in value:
+            value = {**value, "config": REDACTED}
+        return {
+            key: REDACTED
+            if SENSITIVE_KEY.search(key)
+            else redact(item, secrets, field=key)
+            for key, item in value.items()
+        }
+    if isinstance(value, (list, tuple)):
+        return [redact(item, secrets, field=field) for item in value]
+    if not isinstance(value, str):
+        return (
+            REDACTED if type(value) in (int, float) and str(value) in secrets else value
+        )
+    if value in secrets:
+        return REDACTED
+    # Content/generation IDs remain useful even with short numeric passwords.
+    if re.fullmatch(r"[a-f0-9]{32}|[a-f0-9]{64}", value):
+        return value
+    for secret in sorted(secrets, key=len, reverse=True):
+        if len(secret) >= 4 or field in {
+            "message",
+            "error",
+            "reason",
+            "lines",
+            "logs",
+            "traceback",
+            "stderr",
+            "stdout",
+        }:
+            value = value.replace(secret, REDACTED).replace(
+                quote(secret, safe=""), REDACTED
+            )
+    value = URL_AUTH.sub(r"\1[REDACTED]@", value)
+    value = URL_QUERY.sub(r"\1[REDACTED]", value)
+    return MCP_KEY.sub(REDACTED, BEARER.sub(r"\1[REDACTED]", value))
+
+
+def redact_toml(text, redactor):
+    document = tomlkit.parse(text)
+    sanitized = redactor(document.unwrap())
+
+    def overlay(original, safe):
+        for key, value in safe.items():
+            if isinstance(value, dict):
+                overlay(original[key], value)
+            elif original[key] != value:
+                original[key] = value
+
+    overlay(document, sanitized)
+    return tomlkit.dumps(document)

toolframe_core/scaffold.py

@@ -0,0 +1,142 @@
+"""Render the packaged Notes example into an exclusively created source directory."""
+
+from __future__ import annotations
+
+import ast
+import io
+import json
+import re
+import shutil
+import tokenize
+from pathlib import Path
+
+import tomlkit
+from toolframe_sdk import PluginError
+
+from .config import validate_plugin_id
+from .skills import bundled_skill
+
+
+def _files(directory, prefix=Path()):
+    # Traversable also works when core's resources are not ordinary source files.
+    for child in sorted(directory.iterdir(), key=lambda item: item.name):
+        relative = prefix / child.name
+        if child.is_dir():
+            yield from _files(child, relative)
+        else:
+            yield relative, child.read_bytes()
+
+
+def _python(source, rename):
+    tokens = []
+    for token in tokenize.generate_tokens(io.StringIO(source).readline):
+        if token.type == tokenize.STRING:
+            value = ast.literal_eval(token.string)
+            if isinstance(value, str) and (replacement := rename(value)) != value:
+                token = token._replace(string=json.dumps(replacement))
+        elif token.type == tokenize.NAME:
+            token = token._replace(string=rename(token.string))
+        tokens.append(token)
+    return tokenize.untokenize(tokens)
+
+
+def _render(identifier, package, module):
+    template = dict(_files(bundled_skill().joinpath("assets", "sample-plugin")))
+    document = tomlkit.parse(template[Path("pyproject.toml")].decode("utf-8"))
+    project = document["project"]
+    entries = project["entry-points"]["toolframe.plugins"]
+    if len(entries) != 1:
+        raise ValueError("Bundled template must declare exactly one plugin entry point")
+    original_id, entry = next(iter(entries.items()))
+    original_module = entry.split(".", 1)[0]
+    wheel = document["tool"]["hatch"]["build"]["targets"]["wheel"]
+    if wheel["packages"] != [f"src/{original_module}"]:
+        raise ValueError("Bundled template must declare one src-layout Python package")
+
+    identities = {
+        original_id: identifier,
+        project["name"]: package,
+        original_module: module,
+    }
+    prefixes = {
+        f"{original_id}.": f"{identifier}.",  # Service namespace.
+        f"{original_id}-": f"{identifier}-",  # Streamlit widget keys.
+        f"{original_id}__": f"{identifier}__",  # Qualified MCP tool names.
+        f"{original_module}.": f"{module}.",  # Imports and module:factory references.
+    }
+
+    def rename(value):
+        if value in identities:
+            return identities[value]
+        for before, after in prefixes.items():
+            if value.startswith(before):
+                return after + value[len(before) :]
+        return value
+
+    project["name"] = package
+    project["entry-points"]["toolframe.plugins"] = {identifier: rename(entry)}
+    wheel["packages"] = [f"src/{module}"]
+    template[Path("pyproject.toml")] = tomlkit.dumps(document).encode("utf-8")
+
+    rendered = {}
+    for relative, content in template.items():
+        if relative.parts[:2] == ("src", original_module):
+            relative = Path("src", module, *relative.parts[2:])
+        if relative.suffix == ".py":
+            source = _python(content.decode("utf-8"), rename)
+            compile(source, str(relative), "exec")
+            content = source.encode("utf-8")
+        elif relative.suffix == ".md":
+            # Only code spans name callable tools/packages; prose stays verbatim.
+            source = re.sub(
+                r"`([^`\n]+)`",
+                lambda match: f"`{rename(match[1])}`",
+                content.decode("utf-8"),
+            )
+            content = source.encode("utf-8")
+        rendered[relative] = content
+    return rendered
+
+
+def new_plugin(home: Path, identifier: str, *, output: Path | None = None):
+    validate_plugin_id(identifier)
+    package = f"toolframe-{identifier}"
+    # The existing ID grammar permits a trailing '-', unlike distribution names.
+    if package.endswith("-"):
+        package += "plugin"
+    module = package.replace("-", "_")
+    # Do not resolve the last component: that would follow a destination symlink.
+    destination = (
+        output if output is not None else home / "plugins" / identifier
+    ).absolute()
+    conflict = PluginError(
+        "scaffold_conflict",
+        f"Refusing to overwrite existing target: {destination}",
+        {"path": str(destination)},
+    )
+    if destination.exists() or destination.is_symlink():
+        raise conflict
+    rendered = _render(identifier, package, module)
+    destination.parent.mkdir(parents=True, exist_ok=True)
+    try:
+        # mkdir is exclusive, including an empty directory or dangling symlink
+        # created after the first check. Only our successfully created tree is ours.
+        destination.mkdir()
+    except FileExistsError as error:
+        raise conflict from error
+    try:
+        for relative, content in rendered.items():
+            path = destination / relative
+            path.parent.mkdir(parents=True, exist_ok=True)
+            with path.open("xb") as handle:
+                handle.write(content)
+    except BaseException:
+        shutil.rmtree(destination)
+        raise
+    return {
+        "plugin_id": identifier,
+        "package_name": package,
+        "python_module": module,
+        "path": str(destination),
+        "status": "created",
+    }

toolframe_core/skills.py

@@ -0,0 +1,210 @@
+"""Versioned, project-local agent guidance with protection for customized copies."""
+
+import fcntl
+import hashlib
+import json
+import re
+import shutil
+import tempfile
+import uuid
+import zipfile
+from importlib.resources import as_file, files
+from pathlib import Path
+
+import yaml
+from toolframe_sdk import PluginError
+
+SKILL_NAME = "toolframe-plugin-design"
+RECEIPT = ".toolframe-install.json"
+
+
+def bundled_skill():
+    return files("toolframe_core").joinpath("bundled_skills", SKILL_NAME)
+
+
+def _hashes(directory, *, include_receipt=False, exclude_evals=False):
+    if not directory.is_dir() or directory.is_symlink():
+        return None
+    result = {}
+    for path in sorted(directory.rglob("*")):
+        relative = path.relative_to(directory)
+        if exclude_evals and relative.parts[0] == "evals":
+            continue
+        if path.is_symlink():
+            return None
+        if path.is_file():
+            relative = relative.as_posix()
+            if relative != RECEIPT or include_receipt:
+                result[relative] = hashlib.sha256(path.read_bytes()).hexdigest()
+        elif not path.is_dir():
+            return None
+    return result
+
+
+def _record(destination):
+    try:
+        value = json.loads((destination / RECEIPT).read_text())
+        if (
+            value["format"] == 1
+            and value["name"] == SKILL_NAME
+            and re.fullmatch(r"\d+\.\d+\.\d+", value["version"])
+            and isinstance(value["files"], dict)
+            and value["files"]
+            and all(
+                isinstance(k, str) and isinstance(v, str)
+                for k, v in value["files"].items()
+            )
+        ):
+            return value
+    except OSError, ValueError, TypeError, KeyError:
+        pass
+    return None
+
+
+def _publish(source, destination, record, expected, *, backup):
+    staging = Path(tempfile.mkdtemp(prefix=f".{SKILL_NAME}-", dir=destination.parent))
+    previous = staging / "previous"
+    published = False
+    backup_path = None
+    try:
+        candidate = staging / "candidate"
+        candidate.mkdir()
+        # Publish exactly the files in the receipt, excluding development evals
+        # even when this installer runs directly from a source checkout.
+        for relative in record["files"]:
+            target = candidate / relative
+            target.parent.mkdir(parents=True, exist_ok=True)
+            shutil.copy2(source / relative, target)
+        (candidate / RECEIPT).write_text(json.dumps(record, indent=2) + "\n")
+        if destination.exists() or destination.is_symlink():
+            if (
+                expected is None
+                or _hashes(destination, include_receipt=True) != expected
+            ):
+                raise PluginError(
+                    "skill_conflict",
+                    "Skill changed during installation; retry for a fresh comparison.",
+                )
+            destination.rename(previous)
+            if _hashes(previous, include_receipt=True) != expected:
+                raise PluginError(
+                    "skill_conflict",
+                    "Skill changed during installation; the original was retained.",
+                )
+            if backup:
+                backup_path = (
+                    destination.parent / f".{SKILL_NAME}-backup-{uuid.uuid4().hex}.zip"
+                )
+                with zipfile.ZipFile(backup_path, "x", zipfile.ZIP_DEFLATED) as archive:
+                    for path in previous.rglob("*"):
+                        if path.is_file():
+                            archive.write(path, path.relative_to(previous))
+        candidate.rename(destination)
+        published = True
+        return str(backup_path) if backup_path else None
+    finally:
+        if previous.exists() and not published:
+            if destination.exists() or destination.is_symlink():
+                # Keep both copies if an external writer claimed the destination.
+                raise PluginError(
+                    "skill_conflict", f"Original skill retained at {previous}"
+                )
+            previous.rename(destination)
+        shutil.rmtree(staging)
+
+
+def install_skill(
+    home: Path, *, output=None, overwrite=False, confirm=None, upgrade_only=False
+):
+    destination = (output or home / ".agents" / "skills" / SKILL_NAME).absolute()
+    destination.parent.mkdir(parents=True, exist_ok=True)
+    # Cooperating launches/installers must compare and publish under one lock.
+    with (destination.parent / f".{SKILL_NAME}.lock").open("a") as lock:
+        fcntl.flock(lock.fileno(), fcntl.LOCK_EX)
+        with as_file(bundled_skill()) as source:
+            document = (source / "SKILL.md").read_text()
+            match = re.match(
+                r"\A---\r?\n(.*?)\r?\n---(?:\r?\n|\Z)", document, re.DOTALL
+            )
+            try:
+                metadata = yaml.safe_load(match[1]) if match else None
+            except yaml.YAMLError as error:
+                raise ValueError(
+                    "Bundled SKILL.md frontmatter is invalid YAML"
+                ) from error
+            version = metadata.get("version") if isinstance(metadata, dict) else None
+            if not isinstance(version, str) or not re.fullmatch(
+                r"\d+\.\d+\.\d+", version
+            ):
+                raise ValueError(
+                    'Bundled SKILL.md frontmatter requires version: "x.x.x"'
+                )
+            record = {
+                "format": 1,
+                "name": SKILL_NAME,
+                "version": version,
+                "files": _hashes(source, exclude_evals=True),
+            }
+            result = {
+                "name": SKILL_NAME,
+                "path": str(destination),
+                "bundled_version": version,
+                "confirmation_required": False,
+            }
+            exists = destination.exists() or destination.is_symlink()
+            current = _hashes(destination) if exists else None
+            if exists and current is None:
+                return {
+                    **result,
+                    "status": "preserved",
+                    "reason": "non_regular_destination",
+                }
+            prior = _record(destination) if exists else None
+            result["previous_version"] = prior["version"] if prior else None
+            if exists and current == record["files"] and prior == record:
+                return {**result, "status": "unchanged"}
+            if upgrade_only and prior == record:
+                return {
+                    **result,
+                    "status": "preserved",
+                    "reason": "customized_current_version",
+                }
+            changed = exists and (prior is None or current != prior["files"])
+            expected = _hashes(destination, include_receipt=True) if exists else None
+            if changed and not overwrite:
+                conflict = {
+                    **result,
+                    "status": "preserved",
+                    "reason": "local_changes" if prior else "unmanaged_copy",
+                    "confirmation_required": True,
+                }
+                if confirm is None:
+                    return conflict
+                if not confirm(conflict):
+                    return {
+                        **conflict,
+                        "confirmation_required": False,
+                        "reason": "user_declined",
+                    }
+            backup = _publish(source, destination, record, expected, backup=changed)
+            return {
+                **result,
+                "status": "updated" if exists else "installed",
+                "backup": backup,
+            }
+
+
+def installation_message(result):
+    if result["status"] == "preserved":
+        message = f"Kept existing skill at {result['path']} ({result['reason']})."
+        if result["confirmation_required"]:
+            message += " Run skill install interactively to review replacement, or use --overwrite after confirmation; --output PATH exports a separate copy."
+        elif result["reason"] == "non_regular_destination":
+            message += " Use --output PATH for a new regular directory."
+        return message
+    message = (
+        f"Skill {result['status']} ({result['bundled_version']}): {result['path']}"
+    )
+    if result.get("backup"):
+        message += f" Previous customized copy: {result['backup']}"
+    return message