agentskill-toolkit 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agentskill_kit/__init__.py +5 -0
- agentskill_kit/__main__.py +6 -0
- agentskill_kit/cli.py +795 -0
- agentskill_kit/config.py +104 -0
- agentskill_kit/evalrun.py +95 -0
- agentskill_kit/export.py +188 -0
- agentskill_kit/forge.py +346 -0
- agentskill_kit/i18n.py +107 -0
- agentskill_kit/install.py +212 -0
- agentskill_kit/lint.py +201 -0
- agentskill_kit/mine.py +291 -0
- agentskill_kit/mockmodel.py +124 -0
- agentskill_kit/providers.py +344 -0
- agentskill_kit/publish.py +96 -0
- agentskill_kit/registry.py +175 -0
- agentskill_kit/runner.py +214 -0
- agentskill_kit/sandbox.py +118 -0
- agentskill_kit/scaffold.py +90 -0
- agentskill_kit/scan.py +303 -0
- agentskill_kit/scoring.py +93 -0
- agentskill_kit/server.py +69 -0
- agentskill_kit/site.py +24 -0
- agentskill_kit/skillfile.py +378 -0
- agentskill_kit/templates/site.html +275 -0
- agentskill_kit/ui.py +111 -0
- agentskill_toolkit-0.1.0.dist-info/METADATA +314 -0
- agentskill_toolkit-0.1.0.dist-info/RECORD +31 -0
- agentskill_toolkit-0.1.0.dist-info/WHEEL +5 -0
- agentskill_toolkit-0.1.0.dist-info/entry_points.txt +3 -0
- agentskill_toolkit-0.1.0.dist-info/licenses/LICENSE +21 -0
- agentskill_toolkit-0.1.0.dist-info/top_level.txt +1 -0
agentskill_kit/scan.py
ADDED
|
@@ -0,0 +1,303 @@
|
|
|
1
|
+
"""Heuristic security scanner for skills.
|
|
2
|
+
|
|
3
|
+
Skills are instructions that an agent follows, often with shell access, so a
|
|
4
|
+
malicious or careless skill can do real damage. This scanner looks for common
|
|
5
|
+
red flags: prompt injection phrasing, hidden unicode, dangerous shell
|
|
6
|
+
patterns, embedded secrets and signs of credential exfiltration. It is a
|
|
7
|
+
heuristic filter, not a guarantee. Review skills from sources you do not trust.
|
|
8
|
+
|
|
9
|
+
Silence a deliberate match by adding ``askit-ignore`` (optionally followed by
|
|
10
|
+
rule ids, e.g. ``askit-ignore: SC103``) on the same line.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import re
|
|
16
|
+
from dataclasses import dataclass, field
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
from typing import Any, Callable, Dict, List, Optional, Pattern
|
|
19
|
+
|
|
20
|
+
from .skillfile import SkillError, iter_skill_files
|
|
21
|
+
|
|
22
|
+
SEVERITY_WEIGHT = {"high": 40, "medium": 15, "low": 3}
|
|
23
|
+
SEVERITY_RANK = {"high": 3, "medium": 2, "low": 1}
|
|
24
|
+
MAX_FILE_BYTES = 2_000_000
|
|
25
|
+
BINARY_SUFFIXES = {".exe", ".dll", ".so", ".dylib", ".bin", ".jar", ".msi", ".dmg", ".app",
|
|
26
|
+
".class", ".o", ".a", ".whl", ".pyd"}
|
|
27
|
+
CODE_SUFFIXES = {".py", ".js", ".ts", ".mjs", ".cjs", ".sh", ".bash", ".zsh", ".ps1", ".rb", ".pl",
|
|
28
|
+
".php", ".bat", ".cmd"}
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@dataclass
|
|
32
|
+
class Rule:
|
|
33
|
+
id: str
|
|
34
|
+
severity: str
|
|
35
|
+
title: str
|
|
36
|
+
pattern: Pattern[str]
|
|
37
|
+
code_only: bool = False
|
|
38
|
+
accept: Optional[Callable[[str], bool]] = None
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _r(pattern: str, flags: int = re.I) -> Pattern[str]:
|
|
42
|
+
return re.compile(pattern, flags)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _is_placeholder(text: str) -> bool:
|
|
46
|
+
low = text.lower()
|
|
47
|
+
return any(w in low for w in ("example", "your", "xxxx", "changeme", "placeholder", "dummy",
|
|
48
|
+
"sample", "<", "redacted", "0000", "1234", "abcd"))
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _public_ip(text: str) -> bool:
|
|
52
|
+
m = re.search(r"(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})", text)
|
|
53
|
+
if not m:
|
|
54
|
+
return False
|
|
55
|
+
a, b = int(m.group(1)), int(m.group(2))
|
|
56
|
+
if a in (0, 10, 127) or (a == 192 and b == 168) or (a == 172 and 16 <= b <= 31) or (a == 169 and b == 254):
|
|
57
|
+
return False
|
|
58
|
+
return True
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
SECRET_PATTERNS: List[Pattern[str]] = [
|
|
62
|
+
_r(r"\b(?:AKIA|ASIA)[0-9A-Z]{16}\b", 0),
|
|
63
|
+
_r(r"\bgh[pousr]_[A-Za-z0-9]{36,}\b", 0),
|
|
64
|
+
_r(r"\bgithub_pat_[A-Za-z0-9_]{50,}\b", 0),
|
|
65
|
+
_r(r"\bsk-ant-[A-Za-z0-9_-]{20,}", 0),
|
|
66
|
+
_r(r"\bsk-or-v1-[A-Za-z0-9]{20,}", 0),
|
|
67
|
+
_r(r"\bsk-proj-[A-Za-z0-9_-]{20,}", 0),
|
|
68
|
+
_r(r"\bsk-[A-Za-z0-9]{32,}\b", 0),
|
|
69
|
+
_r(r"\bgsk_[A-Za-z0-9]{40,}\b", 0),
|
|
70
|
+
_r(r"\bAIza[0-9A-Za-z_-]{35}\b", 0),
|
|
71
|
+
_r(r"\bxox[abprs]-[A-Za-z0-9-]{10,}", 0),
|
|
72
|
+
_r(r"\bnpm_[A-Za-z0-9]{36}\b", 0),
|
|
73
|
+
_r(r"\bpypi-AgEIcHlwaS5vcmc[A-Za-z0-9_-]{20,}", 0),
|
|
74
|
+
]
|
|
75
|
+
|
|
76
|
+
RULES: List[Rule] = [
|
|
77
|
+
# Prompt injection and hidden instructions
|
|
78
|
+
Rule("SC001", "high", "Instruction override phrasing",
|
|
79
|
+
_r(r"\bignore\s+(?:all\s+|any\s+|the\s+)*(?:previous|prior|above|earlier|preceding)\s+"
|
|
80
|
+
r"(?:instructions?|prompts?|rules|messages|directions)")),
|
|
81
|
+
Rule("SC002", "high", "Attempt to discard system rules",
|
|
82
|
+
_r(r"\bdisregard\s+(?:all\s+|any\s+)*(?:your|the|previous|prior)\s+(?:system\s+)?"
|
|
83
|
+
r"(?:prompt|instructions?|rules|guidelines)")),
|
|
84
|
+
Rule("SC003", "high", "Instruction to hide actions from the user",
|
|
85
|
+
_r(r"\b(?:do\s+not|don't|never)\s+(?:tell|inform|mention|reveal|show|notify)\s+(?:this\s+)?(?:to\s+)?the\s+user\b"
|
|
86
|
+
r"|\bwithout\s+(?:telling|informing|notifying|alerting)\s+the\s+user\b"
|
|
87
|
+
r"|\bhide\s+(?:this|it)\s+from\s+the\s+user\b")),
|
|
88
|
+
Rule("SC004", "medium", "Jailbreak style phrasing",
|
|
89
|
+
_r(r"\b(?:developer\s+mode|jailbreak(?:ed)?|do\s+anything\s+now|you\s+are\s+now\s+(?:in\s+)?"
|
|
90
|
+
r"(?:a\s+|an\s+)?(?:unrestricted|unfiltered|dan)\b)")),
|
|
91
|
+
Rule("SC005", "medium", "Asks the agent to reveal its hidden prompt",
|
|
92
|
+
_r(r"\b(?:reveal|print|output|repeat|leak|dump)\s+(?:your|the)\s+(?:system\s+prompt|"
|
|
93
|
+
r"hidden\s+instructions|initial\s+instructions)")),
|
|
94
|
+
Rule("SC006", "high", "Invisible or bidirectional unicode characters",
|
|
95
|
+
re.compile("[\u200b-\u200f\u202a-\u202e\u2060-\u2064\u2066-\u2069\U000e0000-\U000e007f]")),
|
|
96
|
+
Rule("SC007", "medium", "Instruction hidden inside an HTML comment",
|
|
97
|
+
_r(r"<!--(?:(?!-->).){0,400}?\b(?:ignore|you\s+must|always\s+run|execute|exfiltrate|send\s+the)\b"
|
|
98
|
+
r"(?:(?!-->).){0,400}?-->", re.I | re.S)),
|
|
99
|
+
# Dangerous commands
|
|
100
|
+
Rule("SC101", "high", "Downloads and pipes straight into a shell",
|
|
101
|
+
_r(r"\b(?:curl|wget)\b[^\n|;]*\|\s*(?:sudo\s+)?(?:ba|z|da|k)?sh\b"
|
|
102
|
+
r"|\b(?:iwr|irm|invoke-webrequest|invoke-restmethod)\b[^\n|]*\|\s*(?:iex|invoke-expression)\b")),
|
|
103
|
+
Rule("SC102", "high", "Recursive delete of a broad path",
|
|
104
|
+
_r(r"\brm\s+-(?:[a-z]*r[a-z]*f|[a-z]*f[a-z]*r)[a-z]*\s+(?:--no-preserve-root\s+)?"
|
|
105
|
+
r"(?:/|~|\$HOME|\$\{HOME\}|\*)(?:\s|$|/)")),
|
|
106
|
+
Rule("SC103", "medium", "Uses sudo or elevated privileges",
|
|
107
|
+
_r(r"(?:^|[;&|(`$]|\b(?:run|then|and|exec)\b)[ \t]*sudo[ \t]+[a-z-]", re.I | re.M)),
|
|
108
|
+
Rule("SC104", "medium", "World-writable permissions", _r(r"\bchmod\s+(?:-R\s+)?(?:777|a\+rwx|0777)\b")),
|
|
109
|
+
Rule("SC105", "high", "Decodes and runs obfuscated payload",
|
|
110
|
+
_r(r"\bbase64\s+(?:-d|--decode)\b[^\n|]*\|\s*(?:ba|z)?sh\b"
|
|
111
|
+
r"|\b(?:powershell|pwsh)\b[^\n]*\s-e(?:nc|ncodedcommand)?\s+[A-Za-z0-9+/=]{20,}")),
|
|
112
|
+
Rule("SC106", "high", "Reverse shell pattern",
|
|
113
|
+
_r(r"/dev/tcp/|\bnc(?:at)?\s+(?:-[a-z]*e|-c)\b|\bbash\s+-i\s*>&|\bmkfifo\b[^\n]*\bnc\b")),
|
|
114
|
+
Rule("SC107", "medium", "Dynamic code execution (eval/exec)",
|
|
115
|
+
_r(r"\b(?:eval|exec)\s*\("), code_only=True),
|
|
116
|
+
Rule("SC108", "low", "Shell execution from code",
|
|
117
|
+
_r(r"\bos\.system\s*\(|\bshell\s*=\s*True\b|\bchild_process\.exec\s*\("), code_only=True),
|
|
118
|
+
Rule("SC109", "high", "Destructive disk or fork bomb command",
|
|
119
|
+
_r(r":\(\)\s*\{\s*:\s*\|\s*:\s*&\s*\}\s*;\s*:|\bdd\s+if=\S+\s+of=/dev/(?:sd|nvme|disk|hd)|\bmkfs\.\w+\s+/dev/")),
|
|
120
|
+
Rule("SC110", "low", "Force push or hard reset",
|
|
121
|
+
_r(r"\bgit\s+push\s+(?:[^\n]*\s)?(?:-f|--force)\b|\bgit\s+reset\s+--hard\b")),
|
|
122
|
+
Rule("SC111", "medium", "Installs a package from an arbitrary URL",
|
|
123
|
+
_r(r"\bpip3?\s+install\s+(?:-[A-Za-z-]+\s+)*(?:https?://|git\+)")),
|
|
124
|
+
# Secrets
|
|
125
|
+
Rule("SC201", "high", "AWS access key", SECRET_PATTERNS[0]),
|
|
126
|
+
Rule("SC202", "high", "GitHub token", _r(SECRET_PATTERNS[1].pattern + "|" + SECRET_PATTERNS[2].pattern, 0)),
|
|
127
|
+
Rule("SC203", "high", "LLM or cloud API key",
|
|
128
|
+
_r("|".join(p.pattern for p in SECRET_PATTERNS[3:9]), 0)),
|
|
129
|
+
Rule("SC204", "high", "Slack, npm or PyPI token", _r("|".join(p.pattern for p in SECRET_PATTERNS[9:]), 0)),
|
|
130
|
+
Rule("SC205", "high", "Private key material",
|
|
131
|
+
_r(r"-----BEGIN (?:RSA |EC |DSA |OPENSSH |PGP )?PRIVATE KEY-----", 0)),
|
|
132
|
+
Rule("SC206", "medium", "Hardcoded credential assignment",
|
|
133
|
+
_r(r"\b(?:api[_-]?key|secret|token|passwd|password)\b[\"']?\s*[:=]\s*[\"'][A-Za-z0-9/+_\-]{16,}[\"']"),
|
|
134
|
+
accept=lambda s: not _is_placeholder(s)),
|
|
135
|
+
# Credential access and exfiltration
|
|
136
|
+
Rule("SC301", "medium", "Reads credential files or keys",
|
|
137
|
+
_r(r"(?:~|\$HOME|%USERPROFILE%|/home/\w+)[/\\]\.(?:ssh|aws|gnupg|kube|docker[/\\]config\.json)\b"
|
|
138
|
+
r"|/etc/(?:shadow|sudoers)\b|\bid_rsa\b|\.netrc\b|\bcredentials\.json\b")),
|
|
139
|
+
Rule("SC302", "low", "Sends data over the network",
|
|
140
|
+
_r(r"\brequests\.(?:post|put|patch)\s*\(|\burllib\.request\.urlopen\s*\(|\bhttp\.client\.\w+\s*\("
|
|
141
|
+
r"|\bcurl\b[^\n]*\s(?:-d|--data\S*|-F|-T|--upload-file)\b|\bfetch\s*\(\s*[\"'`]https?://"
|
|
142
|
+
r"|\bsocket\.socket\s*\("), code_only=True),
|
|
143
|
+
Rule("SC303", "medium", "Suspicious host (raw IP, shortener or request catcher)",
|
|
144
|
+
_r(r"https?://(?:\d{1,3}\.){3}\d{1,3}\b|\b(?:bit\.ly|tinyurl\.com|t\.co|goo\.gl|is\.gd)/"
|
|
145
|
+
r"|\b(?:pastebin\.com|webhook\.site|requestbin\.\w+|ngrok\.(?:io|app|dev)|interact\.sh|"
|
|
146
|
+
r"burpcollaborator\.net|discord(?:app)?\.com/api/webhooks)\b"),
|
|
147
|
+
accept=lambda s: _public_ip(s) if re.match(r"https?://\d", s) else True),
|
|
148
|
+
Rule("SC401", "medium", "Persistence mechanism (shell rc, cron, ssh keys)",
|
|
149
|
+
_r(r"\bauthorized_keys\b|\bcrontab\s+-|\blaunchctl\s+load\b|\bsystemctl\s+enable\b"
|
|
150
|
+
r"|>>\s*~?/?\.(?:bashrc|zshrc|profile|bash_profile)\b|\bschtasks\s+/create\b")),
|
|
151
|
+
]
|
|
152
|
+
|
|
153
|
+
RULES_BY_ID = {r.id: r for r in RULES}
|
|
154
|
+
IGNORE_RE = re.compile(r"askit-ignore(?::\s*([A-Z0-9, ]+))?")
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
@dataclass
|
|
158
|
+
class Finding:
|
|
159
|
+
rule: str
|
|
160
|
+
severity: str
|
|
161
|
+
message: str
|
|
162
|
+
file: str
|
|
163
|
+
line: int
|
|
164
|
+
snippet: str = ""
|
|
165
|
+
|
|
166
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
167
|
+
return {"rule": self.rule, "severity": self.severity, "message": self.message,
|
|
168
|
+
"file": self.file, "line": self.line, "snippet": self.snippet}
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
@dataclass
|
|
172
|
+
class ScanResult:
|
|
173
|
+
skill: str
|
|
174
|
+
path: str
|
|
175
|
+
findings: List[Finding] = field(default_factory=list)
|
|
176
|
+
files_scanned: int = 0
|
|
177
|
+
|
|
178
|
+
def count(self, severity: str) -> int:
|
|
179
|
+
return sum(1 for f in self.findings if f.severity == severity)
|
|
180
|
+
|
|
181
|
+
@property
|
|
182
|
+
def risk(self) -> int:
|
|
183
|
+
return min(100, sum(SEVERITY_WEIGHT[f.severity] for f in self.findings))
|
|
184
|
+
|
|
185
|
+
@property
|
|
186
|
+
def safety(self) -> int:
|
|
187
|
+
return 100 - self.risk
|
|
188
|
+
|
|
189
|
+
@property
|
|
190
|
+
def verdict(self) -> str:
|
|
191
|
+
if self.count("high"):
|
|
192
|
+
return "risky"
|
|
193
|
+
if self.count("medium"):
|
|
194
|
+
return "review"
|
|
195
|
+
return "clean"
|
|
196
|
+
|
|
197
|
+
def worst(self) -> str:
|
|
198
|
+
for sev in ("high", "medium", "low"):
|
|
199
|
+
if self.count(sev):
|
|
200
|
+
return sev
|
|
201
|
+
return "none"
|
|
202
|
+
|
|
203
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
204
|
+
return {"skill": self.skill, "path": self.path, "verdict": self.verdict, "safety": self.safety,
|
|
205
|
+
"high": self.count("high"), "medium": self.count("medium"), "low": self.count("low"),
|
|
206
|
+
"files_scanned": self.files_scanned, "findings": [f.to_dict() for f in self.findings]}
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def _ignored(line_text: str, rule_id: str) -> bool:
|
|
210
|
+
m = IGNORE_RE.search(line_text)
|
|
211
|
+
if not m:
|
|
212
|
+
return False
|
|
213
|
+
ids = m.group(1)
|
|
214
|
+
if not ids:
|
|
215
|
+
return True
|
|
216
|
+
return rule_id in {p.strip() for p in ids.split(",")}
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def scan_text(text: str, filename: str = "SKILL.md") -> List[Finding]:
|
|
220
|
+
suffix = Path(filename).suffix.lower()
|
|
221
|
+
is_code = suffix in CODE_SUFFIXES
|
|
222
|
+
lines = text.split("\n")
|
|
223
|
+
findings: List[Finding] = []
|
|
224
|
+
for rule in RULES:
|
|
225
|
+
if rule.code_only and not is_code:
|
|
226
|
+
continue
|
|
227
|
+
for m in rule.pattern.finditer(text):
|
|
228
|
+
matched = m.group(0)
|
|
229
|
+
if rule.accept and not rule.accept(matched):
|
|
230
|
+
continue
|
|
231
|
+
line_no = text.count("\n", 0, m.start()) + 1
|
|
232
|
+
line_text = lines[line_no - 1] if line_no - 1 < len(lines) else ""
|
|
233
|
+
if _ignored(line_text, rule.id):
|
|
234
|
+
continue
|
|
235
|
+
snippet = line_text.strip()
|
|
236
|
+
if rule.id == "SC006":
|
|
237
|
+
snippet = "U+%04X" % ord(matched[0])
|
|
238
|
+
elif rule.id.startswith("SC2") and len(matched) > 8:
|
|
239
|
+
snippet = matched[:4] + "..." + matched[-2:]
|
|
240
|
+
if any(f.rule == rule.id and f.line == line_no for f in findings):
|
|
241
|
+
continue
|
|
242
|
+
findings.append(Finding(rule.id, rule.severity, rule.title, filename, line_no, snippet[:120]))
|
|
243
|
+
# Reading credentials and sending data in the same file is a stronger signal.
|
|
244
|
+
ids = {f.rule for f in findings}
|
|
245
|
+
if "SC301" in ids and "SC302" in ids:
|
|
246
|
+
first = next(f for f in findings if f.rule == "SC301")
|
|
247
|
+
findings.append(Finding("SC310", "high", "Reads credentials and sends data over the network",
|
|
248
|
+
filename, first.line, first.snippet))
|
|
249
|
+
return findings
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def scan_skill(target: Any) -> ScanResult:
|
|
253
|
+
base = Path(target)
|
|
254
|
+
if base.is_file():
|
|
255
|
+
base = base.parent
|
|
256
|
+
if not base.is_dir():
|
|
257
|
+
raise SkillError("not found: %s" % target)
|
|
258
|
+
res = ScanResult(skill=base.name, path=str(base))
|
|
259
|
+
for rel in iter_skill_files(base):
|
|
260
|
+
full = base / rel
|
|
261
|
+
name = rel.as_posix()
|
|
262
|
+
if full.suffix.lower() in BINARY_SUFFIXES:
|
|
263
|
+
res.findings.append(Finding("SC601", "medium", "Compiled or binary file bundled in skill", name, 0))
|
|
264
|
+
continue
|
|
265
|
+
try:
|
|
266
|
+
size = full.stat().st_size
|
|
267
|
+
except OSError:
|
|
268
|
+
continue
|
|
269
|
+
if size > MAX_FILE_BYTES:
|
|
270
|
+
res.findings.append(Finding("SC602", "low", "Very large file (not scanned)", name, 0))
|
|
271
|
+
continue
|
|
272
|
+
try:
|
|
273
|
+
data = full.read_bytes()
|
|
274
|
+
except OSError:
|
|
275
|
+
continue
|
|
276
|
+
if b"\x00" in data[:4096]:
|
|
277
|
+
continue
|
|
278
|
+
text = data.decode("utf-8", errors="replace")
|
|
279
|
+
res.files_scanned += 1
|
|
280
|
+
res.findings.extend(scan_text(text, name))
|
|
281
|
+
res.findings.sort(key=lambda f: (-SEVERITY_RANK[f.severity], f.file, f.line))
|
|
282
|
+
return res
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
def exceeds(result: ScanResult, fail_on: str) -> bool:
|
|
286
|
+
"""True when the scan has a finding at or above the given severity."""
|
|
287
|
+
if fail_on == "never":
|
|
288
|
+
return False
|
|
289
|
+
rank = SEVERITY_RANK.get(fail_on, 3)
|
|
290
|
+
return any(SEVERITY_RANK[f.severity] >= rank for f in result.findings)
|
|
291
|
+
|
|
292
|
+
|
|
293
|
+
def redact(text: str) -> str:
|
|
294
|
+
"""Mask secrets in free text (used when showing shell history)."""
|
|
295
|
+
out = text
|
|
296
|
+
for pat in SECRET_PATTERNS:
|
|
297
|
+
out = pat.sub("<redacted>", out)
|
|
298
|
+
out = re.sub(r"(?i)(--?(?:password|passwd|pass|token|secret|api[-_]?key|key)(?:=|\s+))(\S+)", r"\1<redacted>", out)
|
|
299
|
+
out = re.sub(r"\b([A-Z][A-Z0-9_]*(?:KEY|TOKEN|SECRET|PASSWORD|PASSWD)[A-Z0-9_]*)=(\S+)", r"\1=<redacted>", out)
|
|
300
|
+
out = re.sub(r"(\bmysql(?:dump|admin)?\b[^\n]*?\s-p)(\S+)", r"\1<redacted>", out)
|
|
301
|
+
out = re.sub(r"(?i)(authorization:\s*(?:bearer|basic|token)\s+)\S+", r"\1<redacted>", out)
|
|
302
|
+
out = re.sub(r"(://[^/\s:@]+:)[^@\s/]+(@)", r"\1<redacted>\2", out)
|
|
303
|
+
return out
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
"""Combine lint, scan and test results into a single quality score."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
from dataclasses import dataclass, field
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Any, Dict, Optional
|
|
9
|
+
|
|
10
|
+
from . import runner
|
|
11
|
+
from .lint import LintResult, lint_skill
|
|
12
|
+
from .scan import ScanResult, scan_skill
|
|
13
|
+
from .skillfile import SkillError, load_skill
|
|
14
|
+
|
|
15
|
+
WEIGHTS_TESTED = {"lint": 0.30, "safety": 0.25, "tests": 0.35, "docs": 0.10}
|
|
16
|
+
WEIGHTS_UNTESTED = {"lint": 0.45, "safety": 0.35, "docs": 0.20}
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@dataclass
|
|
20
|
+
class Quality:
|
|
21
|
+
skill: str
|
|
22
|
+
score: float
|
|
23
|
+
grade: str
|
|
24
|
+
tested: bool
|
|
25
|
+
breakdown: Dict[str, float] = field(default_factory=dict)
|
|
26
|
+
lint: Optional[LintResult] = None
|
|
27
|
+
scan: Optional[ScanResult] = None
|
|
28
|
+
tests: Optional[runner.TestReport] = None
|
|
29
|
+
|
|
30
|
+
@property
|
|
31
|
+
def passes_gate(self) -> bool:
|
|
32
|
+
return bool(self.lint and not self.lint.errors and self.scan and self.scan.count("high") == 0
|
|
33
|
+
and not (self.tests and self.tests.has_tests and self.tests.rate < 1.0))
|
|
34
|
+
|
|
35
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
36
|
+
return {"skill": self.skill, "score": round(self.score, 3), "grade": self.grade,
|
|
37
|
+
"tested": self.tested, "breakdown": {k: round(v, 3) for k, v in self.breakdown.items()},
|
|
38
|
+
"lint": self.lint.to_dict() if self.lint else None,
|
|
39
|
+
"scan": self.scan.to_dict() if self.scan else None,
|
|
40
|
+
"tests": self.tests.to_dict() if self.tests else None}
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def grade_for(score: float) -> str:
|
|
44
|
+
for cutoff, letter in ((0.9, "A"), (0.8, "B"), (0.7, "C"), (0.5, "D")):
|
|
45
|
+
if score >= cutoff:
|
|
46
|
+
return letter
|
|
47
|
+
return "F"
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def docs_score(skill_dir: Path) -> float:
|
|
51
|
+
try:
|
|
52
|
+
skill = load_skill(skill_dir)
|
|
53
|
+
except SkillError:
|
|
54
|
+
return 0.0
|
|
55
|
+
body = skill.body
|
|
56
|
+
lines = [ln for ln in body.splitlines() if ln.strip()]
|
|
57
|
+
pts = 0.0
|
|
58
|
+
if len(skill.description) >= 60:
|
|
59
|
+
pts += 0.3
|
|
60
|
+
if re.search(r"^#{1,3}\s+\S", body, re.M):
|
|
61
|
+
pts += 0.2
|
|
62
|
+
if re.search(r"\bexamples?\b", body, re.I) or "```" in body:
|
|
63
|
+
pts += 0.2
|
|
64
|
+
if 12 <= len(lines) <= 500:
|
|
65
|
+
pts += 0.2
|
|
66
|
+
if skill.license or skill.version or skill.tags:
|
|
67
|
+
pts += 0.1
|
|
68
|
+
return min(1.0, pts)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def evaluate_skill(skill_dir: Any, run_tests: bool = True, timeout: float = 20.0) -> Quality:
|
|
72
|
+
base = Path(skill_dir).resolve()
|
|
73
|
+
lint = lint_skill(base)
|
|
74
|
+
scan = scan_skill(base) if base.is_dir() else None
|
|
75
|
+
tests = runner.run_skill_tests(base, timeout=timeout) if (run_tests and base.is_dir()) else None
|
|
76
|
+
tested = bool(tests and tests.has_tests)
|
|
77
|
+
parts: Dict[str, float] = {
|
|
78
|
+
"lint": lint.score / 100.0,
|
|
79
|
+
"safety": (scan.safety / 100.0) if scan else 0.0,
|
|
80
|
+
"docs": docs_score(base) if base.is_dir() else 0.0,
|
|
81
|
+
}
|
|
82
|
+
if tested and tests is not None:
|
|
83
|
+
parts["tests"] = tests.rate
|
|
84
|
+
weights = WEIGHTS_TESTED
|
|
85
|
+
else:
|
|
86
|
+
weights = WEIGHTS_UNTESTED
|
|
87
|
+
score = sum(parts[k] * w for k, w in weights.items())
|
|
88
|
+
if lint.errors:
|
|
89
|
+
score = min(score, 0.59)
|
|
90
|
+
if scan and scan.count("high"):
|
|
91
|
+
score = min(score, 0.49)
|
|
92
|
+
return Quality(skill=lint.skill, score=score, grade=grade_for(score), tested=tested,
|
|
93
|
+
breakdown=parts, lint=lint, scan=scan, tests=tests)
|
agentskill_kit/server.py
ADDED
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
"""Local web UI for browsing the skills on this machine."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import webbrowser
|
|
7
|
+
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import Any, Callable, Dict, List, Optional
|
|
10
|
+
|
|
11
|
+
from . import registry
|
|
12
|
+
from .site import render_site
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def default_roots() -> List[Path]:
|
|
16
|
+
cwd = Path.cwd()
|
|
17
|
+
cands = [cwd / "skills", cwd / ".claude" / "skills", cwd / ".hermes" / "skills",
|
|
18
|
+
Path.home() / ".claude" / "skills", Path.home() / ".hermes" / "skills"]
|
|
19
|
+
out, seen = [], set()
|
|
20
|
+
for c in cands:
|
|
21
|
+
if c.is_dir() and str(c.resolve()) not in seen:
|
|
22
|
+
out.append(c)
|
|
23
|
+
seen.add(str(c.resolve()))
|
|
24
|
+
return out
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def make_handler(build: Callable[[], Dict[str, Any]]):
|
|
28
|
+
class Handler(BaseHTTPRequestHandler):
|
|
29
|
+
def log_message(self, fmt, *args): # keep the terminal quiet
|
|
30
|
+
return
|
|
31
|
+
|
|
32
|
+
def _send(self, code: int, body: bytes, ctype: str) -> None:
|
|
33
|
+
self.send_response(code)
|
|
34
|
+
self.send_header("Content-Type", ctype)
|
|
35
|
+
self.send_header("Content-Length", str(len(body)))
|
|
36
|
+
self.send_header("Cache-Control", "no-store")
|
|
37
|
+
self.send_header("X-Content-Type-Options", "nosniff")
|
|
38
|
+
self.end_headers()
|
|
39
|
+
self.wfile.write(body)
|
|
40
|
+
|
|
41
|
+
def do_GET(self):
|
|
42
|
+
path = self.path.split("?", 1)[0]
|
|
43
|
+
if path in ("/", "/index.html"):
|
|
44
|
+
self._send(200, render_site(build(), "Local skills").encode("utf-8"), "text/html; charset=utf-8")
|
|
45
|
+
elif path == "/index.json":
|
|
46
|
+
self._send(200, json.dumps(build(), ensure_ascii=False).encode("utf-8"), "application/json")
|
|
47
|
+
else:
|
|
48
|
+
self._send(404, b"not found", "text/plain; charset=utf-8")
|
|
49
|
+
|
|
50
|
+
return Handler
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def serve(roots: List[Path], host: str = "127.0.0.1", port: int = 8765, open_browser: bool = False,
|
|
54
|
+
on_ready: Optional[Callable[[str], None]] = None) -> None:
|
|
55
|
+
def build() -> Dict[str, Any]:
|
|
56
|
+
return registry.build_index(roots, name="local skills", run_tests=False, relative_to=Path.cwd())
|
|
57
|
+
|
|
58
|
+
server = ThreadingHTTPServer((host, port), make_handler(build))
|
|
59
|
+
url = "http://%s:%d/" % (host, server.server_address[1])
|
|
60
|
+
if on_ready:
|
|
61
|
+
on_ready(url)
|
|
62
|
+
if open_browser:
|
|
63
|
+
webbrowser.open(url)
|
|
64
|
+
try:
|
|
65
|
+
server.serve_forever()
|
|
66
|
+
except KeyboardInterrupt:
|
|
67
|
+
pass
|
|
68
|
+
finally:
|
|
69
|
+
server.server_close()
|
agentskill_kit/site.py
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
"""Render the registry web page (static file, used by `registry site` and `serve`)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Any, Dict
|
|
8
|
+
|
|
9
|
+
TEMPLATE = Path(__file__).with_name("templates") / "site.html"
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def render_site(index: Dict[str, Any], title: str = "agentskill-kit skill registry") -> str:
|
|
13
|
+
html = TEMPLATE.read_text(encoding="utf-8")
|
|
14
|
+
data = json.dumps(index, ensure_ascii=False).replace("</", "<\\/").replace("\u2028", "\\u2028").replace("\u2029", "\\u2029")
|
|
15
|
+
return html.replace("__TITLE__", title.replace("<", "<")).replace("__DATA__", data)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def write_site(index: Dict[str, Any], out_dir: Any, title: str = "agentskill-kit skill registry") -> Path:
|
|
19
|
+
out = Path(out_dir)
|
|
20
|
+
out.mkdir(parents=True, exist_ok=True)
|
|
21
|
+
(out / "index.html").write_text(render_site(index, title), encoding="utf-8")
|
|
22
|
+
(out / "index.json").write_text(json.dumps(index, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
|
23
|
+
(out / ".nojekyll").write_text("", encoding="utf-8")
|
|
24
|
+
return out / "index.html"
|