agentskill-toolkit 0.1.0__py3-none-any.whl

agentskill_kit/runner.py

@@ -0,0 +1,214 @@
+"""Run the tests that ship with a skill.
+
+Two formats are supported and can be combined:
+
+* ``tests/cases.json``: declarative command line cases for a script.
+* ``tests/test_*.py``: standard unittest modules, run from the skill root.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+from . import sandbox
+from .scan import scan_text, SEVERITY_RANK
+
+
+@dataclass
+class CaseResult:
+    name: str
+    kind: str
+    passed: bool
+    detail: str = ""
+    duration: float = 0.0
+    n_passed: Optional[int] = None
+    n_total: int = 1
+
+    def __post_init__(self) -> None:
+        if self.n_passed is None:
+            self.n_passed = 1 if self.passed else 0
+
+
+@dataclass
+class TestReport:
+    __test__ = False  # keep pytest from collecting this class
+
+    skill: str
+    has_tests: bool = False
+    cases: List[CaseResult] = field(default_factory=list)
+    refused: str = ""
+
+    @property
+    def passed(self) -> int:
+        return sum(int(c.n_passed or 0) for c in self.cases)
+
+    @property
+    def total(self) -> int:
+        return sum(c.n_total for c in self.cases)
+
+    @property
+    def rate(self) -> float:
+        if self.refused:
+            return 0.0
+        return (self.passed / self.total) if self.total else 0.0
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {"skill": self.skill, "has_tests": self.has_tests, "passed": self.passed,
+                "total": self.total, "refused": self.refused,
+                "cases": [{"name": c.name, "kind": c.kind, "passed": c.passed, "detail": c.detail}
+                          for c in self.cases]}
+
+
+def has_tests(skill_dir: Path) -> bool:
+    t = skill_dir / "tests"
+    return t.is_dir() and ((t / "cases.json").exists() or any(t.glob("test_*.py")))
+
+
+def _safe_rel(rel: str) -> bool:
+    p = Path(rel)
+    return not p.is_absolute() and ".." not in p.parts and bool(rel)
+
+
+def _as_list(v: Any) -> List[str]:
+    if v is None:
+        return []
+    return [str(x) for x in v] if isinstance(v, list) else [str(v)]
+
+
+def _subset(expected: Any, actual: Any) -> bool:
+    if isinstance(expected, dict):
+        return isinstance(actual, dict) and all(k in actual and _subset(v, actual[k]) for k, v in expected.items())
+    if isinstance(expected, list):
+        return isinstance(actual, list) and len(expected) == len(actual) and all(
+            _subset(e, a) for e, a in zip(expected, actual))
+    return expected == actual
+
+
+def _check_case(case: Dict[str, Any], res: sandbox.RunResult) -> str:
+    """Return an empty string when the case passes, otherwise a reason."""
+    if res.error:
+        return "could not start: " + res.error
+    if res.timed_out:
+        return "timed out"
+    want_exit = case.get("expect_exit", 0)
+    if want_exit is not None and res.exit_code != want_exit:
+        tail = (res.stderr or res.stdout).strip().splitlines()[-1:] or [""]
+        return "exit code %s, expected %s (%s)" % (res.exit_code, want_exit, tail[0][:120])
+    for s in _as_list(case.get("stdout_contains")):
+        if s not in res.stdout:
+            return "stdout does not contain %r" % s
+    for s in _as_list(case.get("stdout_not_contains")):
+        if s in res.stdout:
+            return "stdout unexpectedly contains %r" % s
+    for s in _as_list(case.get("stderr_contains")):
+        if s not in res.stderr:
+            return "stderr does not contain %r" % s
+    for rx in _as_list(case.get("stdout_regex")):
+        if not re.search(rx, res.stdout, re.M):
+            return "stdout does not match /%s/" % rx
+    if "stdout_json" in case:
+        try:
+            actual = json.loads(res.stdout)
+        except ValueError:
+            return "stdout is not valid JSON"
+        if not _subset(case["stdout_json"], actual):
+            return "JSON output differs from expected"
+    return ""
+
+
+def _run_cases(root: Path, spec: Dict[str, Any], timeout: float) -> List[CaseResult]:
+    """Run declarative cases. Each case gets its own empty working directory."""
+    import shutil
+    import tempfile
+
+    entry = spec.get("entry")
+    out: List[CaseResult] = []
+    for idx, case in enumerate(spec.get("cases", [])):
+        name = str(case.get("name") or "case %d" % (idx + 1))
+        kind = str(case.get("kind") or "case")
+        script = case.get("entry") or entry
+        if not script or not _safe_rel(str(script)) or not (root / script).is_file():
+            out.append(CaseResult(name, kind, False, "entry script not found: %s" % script))
+            continue
+        bad = next((f for f in (case.get("files") or {}) if not _safe_rel(f)), None)
+        if bad:
+            out.append(CaseResult(name, kind, False, "unsafe file path in case: %s" % bad))
+            continue
+        workdir = Path(tempfile.mkdtemp(prefix="askit-case-"))
+        try:
+            for rel, content in (case.get("files") or {}).items():
+                p = workdir / rel
+                p.parent.mkdir(parents=True, exist_ok=True)
+                p.write_text(str(content), encoding="utf-8")
+            args = [str((root / script).resolve())] + [str(a) for a in case.get("args", [])]
+            res = sandbox.run_python(args, workdir, stdin=str(case.get("stdin", "")),
+                                     timeout=float(case.get("timeout", timeout)))
+        finally:
+            shutil.rmtree(workdir, ignore_errors=True)
+        reason = _check_case(case, res)
+        out.append(CaseResult(name, kind, not reason, reason, res.duration))
+    return out
+
+
+def _run_unittests(root: Path, timeout: float) -> Optional[CaseResult]:
+    tests_dir = root / "tests"
+    if not any(tests_dir.glob("test_*.py")):
+        return None
+    res = sandbox.run_python(["-m", "unittest", "discover", "-s", "tests", "-p", "test_*.py"],
+                             root, timeout=timeout * 3)
+    text = res.stderr + "\n" + res.stdout
+    m = re.search(r"Ran (\d+) tests?", text)
+    if not m:
+        return CaseResult("unittest", "unittest", False, "could not run: " + text.strip()[-200:], res.duration, 0, 1)
+    total = int(m.group(1))
+    failures = int(next(iter(re.findall(r"failures=(\d+)", text)), 0))
+    errors = int(next(iter(re.findall(r"errors=(\d+)", text)), 0))
+    skipped = int(next(iter(re.findall(r"skipped=(\d+)", text)), 0))
+    passed = max(0, total - failures - errors)
+    ok = failures == 0 and errors == 0
+    detail = "" if ok else "%d failure(s), %d error(s)" % (failures, errors)
+    return CaseResult("unittest (%d tests%s)" % (total, ", %d skipped" % skipped if skipped else ""),
+                      "unittest", ok, detail, res.duration, passed, max(total, 1))
+
+
+def run_skill_tests(skill_dir: Any, timeout: float = 20.0, allow_unsafe: bool = False) -> TestReport:
+    base = Path(skill_dir).resolve()
+    report = TestReport(skill=base.name)
+    if not has_tests(base):
+        return report
+    report.has_tests = True
+
+    if not allow_unsafe:
+        for py in sorted(base.rglob("*.py")):
+            if "__pycache__" in py.parts:
+                continue
+            try:
+                findings = scan_text(py.read_text(encoding="utf-8", errors="replace"), py.name)
+            except OSError:
+                continue
+            high = [f for f in findings if SEVERITY_RANK[f.severity] >= 3]
+            if high:
+                report.refused = "%s has high severity findings (%s); refusing to execute" % (
+                    py.relative_to(base).as_posix(), ", ".join(sorted({f.rule for f in high})))
+                return report
+
+    with sandbox.scratch_copy(base) as root:
+        spec_path = root / "tests" / "cases.json"
+        if spec_path.exists():
+            try:
+                spec = json.loads(spec_path.read_text(encoding="utf-8"))
+            except ValueError as exc:
+                report.cases.append(CaseResult("cases.json", "spec", False, "invalid JSON: %s" % exc))
+                spec = {}
+            if isinstance(spec, dict) and spec.get("cases"):
+                report.cases.extend(_run_cases(root, spec, timeout))
+            elif isinstance(spec, dict) and not report.cases:
+                report.cases.append(CaseResult("cases.json", "spec", False, "no cases defined"))
+        ut = _run_unittests(root, timeout)
+        if ut:
+            report.cases.append(ut)
+    return report

agentskill_kit/sandbox.py

@@ -0,0 +1,118 @@
+"""Run untrusted skill scripts in a throwaway working directory.
+
+This limits accidents (runaway loops, huge output, stray writes, leaked API
+keys in the environment). It is not a security boundary: a hostile script can
+still do anything the current user can. The scanner refuses obviously
+dangerous code before it is run, and you should use containers for real
+isolation when testing skills you do not trust.
+"""
+
+from __future__ import annotations
+
+import os
+import shutil
+import signal
+import subprocess
+import sys
+import tempfile
+import time
+from contextlib import contextmanager
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Dict, Iterator, List, Optional
+
+MAX_OUTPUT = 20_000
+_PASS_ENV = ("PATH", "SYSTEMROOT", "SystemRoot", "COMSPEC", "PATHEXT", "LANG", "LC_ALL", "TZ")
+
+
+@dataclass
+class RunResult:
+    exit_code: int
+    stdout: str
+    stderr: str
+    timed_out: bool = False
+    duration: float = 0.0
+    error: str = ""
+
+
+def clean_env(workdir: Path, extra: Optional[Dict[str, str]] = None) -> Dict[str, str]:
+    env = {k: os.environ[k] for k in _PASS_ENV if k in os.environ}
+    env.update({
+        "HOME": str(workdir), "USERPROFILE": str(workdir),
+        "TEMP": str(workdir), "TMP": str(workdir), "TMPDIR": str(workdir),
+        "PYTHONIOENCODING": "utf-8", "PYTHONDONTWRITEBYTECODE": "1", "PYTHONUTF8": "1",
+        "NO_COLOR": "1",
+    })
+    if extra:
+        env.update(extra)
+    return env
+
+
+def _limit_resources(timeout: float, mem_mb: int):
+    def apply() -> None:
+        try:
+            import resource
+            os.setsid()
+            resource.setrlimit(resource.RLIMIT_CPU, (int(timeout) + 2, int(timeout) + 2))
+            resource.setrlimit(resource.RLIMIT_FSIZE, (50 * 1024 * 1024, 50 * 1024 * 1024))
+            resource.setrlimit(resource.RLIMIT_NOFILE, (256, 256))
+            if sys.platform.startswith("linux"):
+                limit = mem_mb * 1024 * 1024
+                resource.setrlimit(resource.RLIMIT_AS, (limit, limit))
+        except Exception:
+            pass
+    return apply
+
+
+def _trim(text: str) -> str:
+    # Windows consoles turn "\n" into "\r\n"; tests should see the same text everywhere.
+    text = text.replace("\r\n", "\n")
+    return text if len(text) <= MAX_OUTPUT else text[:MAX_OUTPUT] + "\n...[output truncated]"
+
+
+def run_python(args: List[str], cwd: Path, stdin: str = "", timeout: float = 20.0,
+               extra_env: Optional[Dict[str, str]] = None, mem_mb: int = 768) -> RunResult:
+    """Run `python <args>` inside cwd with a clean environment."""
+    cmd = [sys.executable, "-B"] + list(args)
+    start = time.time()
+    kwargs: Dict[str, object] = {}
+    if os.name == "nt":
+        kwargs["creationflags"] = getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0)
+    else:
+        kwargs["preexec_fn"] = _limit_resources(timeout, mem_mb)
+    try:
+        proc = subprocess.Popen(
+            cmd, cwd=str(cwd), env=clean_env(cwd, extra_env), stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs)  # type: ignore[arg-type]
+    except OSError as exc:
+        return RunResult(-1, "", "", error=str(exc))
+    try:
+        out, err = proc.communicate(stdin.encode("utf-8"), timeout=timeout)
+        timed_out = False
+    except subprocess.TimeoutExpired:
+        timed_out = True
+        try:
+            if os.name == "nt":
+                proc.kill()
+            else:
+                os.killpg(proc.pid, signal.SIGKILL)
+        except Exception:
+            proc.kill()
+        out, err = proc.communicate()
+    return RunResult(
+        exit_code=-1 if timed_out else proc.returncode,
+        stdout=_trim(out.decode("utf-8", errors="replace")),
+        stderr=_trim(err.decode("utf-8", errors="replace")),
+        timed_out=timed_out, duration=time.time() - start)
+
+
+@contextmanager
+def scratch_copy(skill_dir: Path) -> Iterator[Path]:
+    """Copy a skill into a temp directory so tests never touch the original."""
+    tmp = Path(tempfile.mkdtemp(prefix="askit-"))
+    target = tmp / skill_dir.name
+    try:
+        shutil.copytree(skill_dir, target, ignore=shutil.ignore_patterns("__pycache__", ".git", "*.pyc"))
+        yield target
+    finally:
+        shutil.rmtree(tmp, ignore_errors=True)

agentskill_kit/scaffold.py

@@ -0,0 +1,90 @@
+"""Create a new skill from a template."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import List, Optional
+
+from .skillfile import dump_frontmatter, slugify
+
+SCRIPT = '''#!/usr/bin/env python3
+"""%(summary)s"""
+import argparse
+import json
+import sys
+
+
+def run(text):
+    """Return a result for the given text."""
+    words = text.split()
+    return {"words": len(words), "characters": len(text)}
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("file", nargs="?", help="input file (reads stdin when omitted)")
+    args = parser.parse_args(argv)
+    if args.file:
+        try:
+            with open(args.file, encoding="utf-8") as fh:
+                text = fh.read()
+        except OSError:
+            print("file not found: %%s" %% args.file, file=sys.stderr)
+            return 2
+    else:
+        text = sys.stdin.read()
+    print(json.dumps(run(text)))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())
+'''
+
+CASES = '''{
+  "entry": "scripts/main.py",
+  "cases": [
+    {"name": "counts words", "kind": "happy", "stdin": "one two three",
+     "stdout_json": {"words": 3, "characters": 13}},
+    {"name": "empty input", "kind": "edge", "stdin": "",
+     "stdout_json": {"words": 0, "characters": 0}},
+    {"name": "missing file", "kind": "error", "args": ["missing.txt"],
+     "expect_exit": 2, "stderr_contains": "file not found"}
+  ]
+}
+'''
+
+
+def create_skill(name: str, parent: Path, description: str = "", with_script: bool = False,
+                 author: str = "", tags: Optional[List[str]] = None, force: bool = False) -> Path:
+    slug = slugify(name)
+    dest = Path(parent) / slug
+    if dest.exists() and any(dest.iterdir()) and not force:
+        raise FileExistsError(str(dest))
+    dest.mkdir(parents=True, exist_ok=True)
+    desc = description.strip() or ("Describe what this skill does in one sentence. Use when the user asks "
+                                   "for the task this skill covers.")
+    meta = {"version": "0.1.0"}
+    if author:
+        meta["author"] = author
+    meta["tags"] = tags or ["general"]
+    fm = dump_frontmatter({"name": slug, "description": desc, "license": "MIT", "metadata": meta})
+    title = slug.replace("-", " ").capitalize()
+    lines = [fm, "# %s\n" % title,
+             "## When to use\n",
+             "- Replace this line with the situations where an agent should pick this skill.\n",
+             "## Steps\n",
+             "1. Replace this step with the first thing the agent should do.",
+             "2. Add the next step. Keep every step concrete and checkable.\n"]
+    if with_script:
+        lines += ["## Script\n", "```bash", "python scripts/main.py path/to/input.txt", "```\n",
+                  "The script prints JSON with the word and character count. Replace it with your own logic.\n"]
+    lines += ["## Examples\n", "Input: describe a typical request.\n", "Output: describe the expected result.\n",
+              "## Errors\n", "- Say what the agent should do when the input is missing or invalid.\n"]
+    (dest / "SKILL.md").write_text("\n".join(lines), encoding="utf-8")
+    if with_script:
+        (dest / "scripts").mkdir(exist_ok=True)
+        (dest / "scripts" / "main.py").write_text(SCRIPT % {"summary": desc.split(".")[0] + "."}, encoding="utf-8")
+        (dest / "tests").mkdir(exist_ok=True)
+        (dest / "tests" / "cases.json").write_text(CASES, encoding="utf-8")
+    return dest