skill-drift 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- skill_drift-0.1.0/LICENSE +21 -0
- skill_drift-0.1.0/PKG-INFO +155 -0
- skill_drift-0.1.0/README.md +144 -0
- skill_drift-0.1.0/pyproject.toml +19 -0
- skill_drift-0.1.0/setup.cfg +4 -0
- skill_drift-0.1.0/src/skill_drift/__init__.py +2 -0
- skill_drift-0.1.0/src/skill_drift/__main__.py +3 -0
- skill_drift-0.1.0/src/skill_drift/audit.py +89 -0
- skill_drift-0.1.0/src/skill_drift/cli.py +34 -0
- skill_drift-0.1.0/src/skill_drift/skills.py +81 -0
- skill_drift-0.1.0/src/skill_drift.egg-info/PKG-INFO +155 -0
- skill_drift-0.1.0/src/skill_drift.egg-info/SOURCES.txt +16 -0
- skill_drift-0.1.0/src/skill_drift.egg-info/dependency_links.txt +1 -0
- skill_drift-0.1.0/src/skill_drift.egg-info/entry_points.txt +2 -0
- skill_drift-0.1.0/src/skill_drift.egg-info/top_level.txt +1 -0
- skill_drift-0.1.0/tests/test_audit.py +88 -0
- skill_drift-0.1.0/tests/test_cli.py +49 -0
- skill_drift-0.1.0/tests/test_discovery.py +60 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 skill-drift contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: skill-drift
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Offline heuristic auditing of skill package claims
|
|
5
|
+
Author: skill-drift contributors
|
|
6
|
+
License: MIT
|
|
7
|
+
Requires-Python: >=3.9
|
|
8
|
+
Description-Content-Type: text/markdown
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Dynamic: license-file
|
|
11
|
+
|
|
12
|
+
# skill-drift
|
|
13
|
+
|
|
14
|
+
`skill-drift` 0.1.0 checks whether an agent skill package's stated paths and
|
|
15
|
+
basic metadata still match the disk. It is deterministic, offline, and uses
|
|
16
|
+
only the Python standard library at runtime: no daemon, LLM, network, or keys.
|
|
17
|
+
A finding means **check this claim**.
|
|
18
|
+
|
|
19
|
+
Requires Python 3.9+. Install with `pip install .`, or install a built wheel.
|
|
20
|
+
|
|
21
|
+
```sh
|
|
22
|
+
skill-drift audit [path] [--json] [--fail-on high|medium] [--ignore G]
|
|
23
|
+
skill-drift --version
|
|
24
|
+
skill-drift --help
|
|
25
|
+
python -m skill_drift audit .
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
Path defaults to the current directory. Repeat `--ignore` for multiple glob
|
|
29
|
+
patterns; patterns match the claim text, apply to all checks, and excluded
|
|
30
|
+
unique findings are counted as ignored.
|
|
31
|
+
|
|
32
|
+
## Discovery and parsing
|
|
33
|
+
|
|
34
|
+
A package is any directory containing a file named exactly `SKILL.md`.
|
|
35
|
+
Discovery recursively includes the supplied directory itself and hidden
|
|
36
|
+
folders such as `.claude/skills` and `.cursor/skills`. It skips `.git`,
|
|
37
|
+
`node_modules`, `dist`, `__pycache__`, `.venv`, and `venv`; directory symlinks
|
|
38
|
+
are not traversed. No packages gives PASS, exit 0, and `no skills found`.
|
|
39
|
+
Unreadable or invalid UTF-8 SKILL.md files produce parse errors.
|
|
40
|
+
|
|
41
|
+
The file must begin with an exact `---` line and close its frontmatter with
|
|
42
|
+
another `---` line. Supported YAML is a subset: `key: value`, quoted or
|
|
43
|
+
unquoted scalar values, and block lists under a key. Blank lines and full-line
|
|
44
|
+
comments are accepted. Nested maps, anchors, tags, flow collections, multiline
|
|
45
|
+
scalars, duplicate keys, and malformed blocks are parse errors. Name and
|
|
46
|
+
description must be scalars. Quoted values are treated literally, without
|
|
47
|
+
YAML escape processing; inline comments are not interpreted.
|
|
48
|
+
|
|
49
|
+
## Checks and severities
|
|
50
|
+
|
|
51
|
+
| Kind | Severity | Trigger |
|
|
52
|
+
| --- | --- | --- |
|
|
53
|
+
| missing-frontmatter | HIGH | No opening frontmatter delimiter |
|
|
54
|
+
| missing-name | HIGH | Missing or empty name |
|
|
55
|
+
| missing-description | HIGH | Missing or empty description |
|
|
56
|
+
| name-mismatch | MEDIUM | Name differs from package directory name |
|
|
57
|
+
| referenced-file | HIGH | Referenced path does not exist |
|
|
58
|
+
| script-not-executable | MEDIUM | Referenced scripts/ file has neither any executable bit nor a shebang |
|
|
59
|
+
| duplicate-skill-name | MEDIUM | Same nonempty name appears in multiple packages; each package is flagged |
|
|
60
|
+
| empty-body | MEDIUM | Stripped body has fewer than 50 characters |
|
|
61
|
+
|
|
62
|
+
Name and description are HIGH by design because they make a skill loadable
|
|
63
|
+
and discoverable. Name matching is a convention, not universal across hosts.
|
|
64
|
+
|
|
65
|
+
Path checks inspect the body after frontmatter, using backticked or bare
|
|
66
|
+
tokens containing `/` and ending in `.py`, `.js`, `.ts`, `.md`, `.json`,
|
|
67
|
+
`.yaml`, `.yml`, `.toml`, `.sh`, `.go`, `.rs`, or `.txt`, plus references
|
|
68
|
+
under `scripts/`, `references/`, `examples/`, `tests/`, `docs/`, and `bin/`.
|
|
69
|
+
URLs beginning http:// or https://, absolute paths, and tokens containing `*`
|
|
70
|
+
are skipped. Paths resolve against the skill directory first, then the audit
|
|
71
|
+
root. A trailing slash requires a directory. Shebang detection checks the
|
|
72
|
+
first two bytes for `#!`; permission checks inspect mode bits.
|
|
73
|
+
|
|
74
|
+
Findings include skill directory relative to the root, file, 1-based line,
|
|
75
|
+
kind, claim, severity, and detail. Metadata diagnostics use the opening lines;
|
|
76
|
+
body path diagnostics use their actual line. Findings deduplicate by
|
|
77
|
+
(skill, line, kind, claim). At most 500 are displayed, ordered by skill then
|
|
78
|
+
line (then kind and claim); truncation and omitted count are explicit.
|
|
79
|
+
Totals and verdict include all unignored findings, including omitted ones.
|
|
80
|
+
Human and JSON reports contain the same counts, per-kind and per-severity
|
|
81
|
+
totals, ignored count, parse-error count, and verdict.
|
|
82
|
+
|
|
83
|
+
## Exit codes
|
|
84
|
+
|
|
85
|
+
| Code | Verdict | Meaning |
|
|
86
|
+
| --- | --- | --- |
|
|
87
|
+
| 0 | PASS | No unignored findings or parse errors |
|
|
88
|
+
| 1 | WARN | Findings exist, all below the threshold |
|
|
89
|
+
| 2 | FAIL | Any finding reaches the threshold, or any parse error |
|
|
90
|
+
|
|
91
|
+
Default threshold is HIGH. With `--fail-on medium`, every finding fails.
|
|
92
|
+
An invalid audit path also produces exit 2. CLI argument errors use argparse's
|
|
93
|
+
exit 2. The warn-only case uses exit 1 even though no finding reaches the
|
|
94
|
+
threshold.
|
|
95
|
+
|
|
96
|
+
## Examples
|
|
97
|
+
|
|
98
|
+
All included examples are synthetic and need no credentials or services.
|
|
99
|
+
|
|
100
|
+
```sh
|
|
101
|
+
skill-drift audit examples/sample-skills/good-skill
|
|
102
|
+
# PASS: 1 skills found; no findings; exit 0
|
|
103
|
+
skill-drift audit examples/sample-skills/broken-skill
|
|
104
|
+
# FAIL: 1 skills found; exit 2
|
|
105
|
+
# HIGH missing-description: description
|
|
106
|
+
# MEDIUM name-mismatch: outdated-skill
|
|
107
|
+
# HIGH referenced-file: scripts/renamed.sh
|
|
108
|
+
# HIGH referenced-file: references/moved.md
|
|
109
|
+
skill-drift audit examples/sample-skills/broken-skill --json
|
|
110
|
+
# Valid JSON: HIGH=3, MEDIUM=1, verdict=FAIL, exit_code=2
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
The good package references an executable `scripts/check.sh`. The broken
|
|
114
|
+
package demonstrates a missing description, mismatched name, missing script,
|
|
115
|
+
and missing reference. The actual report also includes locations and details.
|
|
116
|
+
|
|
117
|
+
## Limitations
|
|
118
|
+
|
|
119
|
+
Checks are heuristic pattern matching, not semantic judgement. Paraphrased
|
|
120
|
+
claims are not checked. Commands are never executed; only their path tokens
|
|
121
|
+
are checked. Paths with spaces and unusual punctuation may be missed or
|
|
122
|
+
misidentified. Existence says nothing about correctness, safety, importability,
|
|
123
|
+
or successful execution. External targets and glob expansions are not checked.
|
|
124
|
+
The YAML subset rejects complex YAML instead of guessing. Filesystem changes
|
|
125
|
+
during a run can affect results; this is a snapshot-style audit, not a monitor.
|
|
126
|
+
|
|
127
|
+
## Differentiation
|
|
128
|
+
|
|
129
|
+
- **instruction-drift** audits instruction files (AGENTS.md, CLAUDE.md,
|
|
130
|
+
MUSE.md, CONTRIBUTING.md) against a repository. This audits skill packages:
|
|
131
|
+
SKILL.md, frontmatter, scripts, and references against the disk.
|
|
132
|
+
- **skill-tax** audits what a SKILL.md costs in tokens. This audits whether
|
|
133
|
+
the skill still works through its checkable package claims.
|
|
134
|
+
- **skill-pin-verify** audits pinning/signatures. This audits the truth of
|
|
135
|
+
package contents through offline heuristics.
|
|
136
|
+
- **frontmatter-guard** audits plugin.json manifests. This audits SKILL.md
|
|
137
|
+
frontmatter.
|
|
138
|
+
- **Heavy runtimes** require more infrastructure. This is an offline stdlib
|
|
139
|
+
CLI with no daemon, no LLM, and no network.
|
|
140
|
+
|
|
141
|
+
## Development
|
|
142
|
+
|
|
143
|
+
```sh
|
|
144
|
+
PYTHONPATH=src python -m unittest discover -s tests -v
|
|
145
|
+
python -m build
|
|
146
|
+
python -m venv /tmp/skill-drift-smoke
|
|
147
|
+
/tmp/skill-drift-smoke/bin/pip install dist/skill_drift-0.1.0-py3-none-any.whl
|
|
148
|
+
/tmp/skill-drift-smoke/bin/skill-drift audit examples/sample-skills/good-skill
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
Tests use unittest, temporary synthetic fixtures, and no network. Setuptools
|
|
152
|
+
and wheel are build-time dependencies; `build` is the development frontend.
|
|
153
|
+
Build artifacts are local only and should be removed before sharing a tree.
|
|
154
|
+
|
|
155
|
+
MIT licensed. See LICENSE.
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
# skill-drift
|
|
2
|
+
|
|
3
|
+
`skill-drift` 0.1.0 checks whether an agent skill package's stated paths and
|
|
4
|
+
basic metadata still match the disk. It is deterministic, offline, and uses
|
|
5
|
+
only the Python standard library at runtime: no daemon, LLM, network, or keys.
|
|
6
|
+
A finding means **check this claim**.
|
|
7
|
+
|
|
8
|
+
Requires Python 3.9+. Install with `pip install .`, or install a built wheel.
|
|
9
|
+
|
|
10
|
+
```sh
|
|
11
|
+
skill-drift audit [path] [--json] [--fail-on high|medium] [--ignore G]
|
|
12
|
+
skill-drift --version
|
|
13
|
+
skill-drift --help
|
|
14
|
+
python -m skill_drift audit .
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
Path defaults to the current directory. Repeat `--ignore` for multiple glob
|
|
18
|
+
patterns; patterns match the claim text, apply to all checks, and excluded
|
|
19
|
+
unique findings are counted as ignored.
|
|
20
|
+
|
|
21
|
+
## Discovery and parsing
|
|
22
|
+
|
|
23
|
+
A package is any directory containing a file named exactly `SKILL.md`.
|
|
24
|
+
Discovery recursively includes the supplied directory itself and hidden
|
|
25
|
+
folders such as `.claude/skills` and `.cursor/skills`. It skips `.git`,
|
|
26
|
+
`node_modules`, `dist`, `__pycache__`, `.venv`, and `venv`; directory symlinks
|
|
27
|
+
are not traversed. No packages gives PASS, exit 0, and `no skills found`.
|
|
28
|
+
Unreadable or invalid UTF-8 SKILL.md files produce parse errors.
|
|
29
|
+
|
|
30
|
+
The file must begin with an exact `---` line and close its frontmatter with
|
|
31
|
+
another `---` line. Supported YAML is a subset: `key: value`, quoted or
|
|
32
|
+
unquoted scalar values, and block lists under a key. Blank lines and full-line
|
|
33
|
+
comments are accepted. Nested maps, anchors, tags, flow collections, multiline
|
|
34
|
+
scalars, duplicate keys, and malformed blocks are parse errors. Name and
|
|
35
|
+
description must be scalars. Quoted values are treated literally, without
|
|
36
|
+
YAML escape processing; inline comments are not interpreted.
|
|
37
|
+
|
|
38
|
+
## Checks and severities
|
|
39
|
+
|
|
40
|
+
| Kind | Severity | Trigger |
|
|
41
|
+
| --- | --- | --- |
|
|
42
|
+
| missing-frontmatter | HIGH | No opening frontmatter delimiter |
|
|
43
|
+
| missing-name | HIGH | Missing or empty name |
|
|
44
|
+
| missing-description | HIGH | Missing or empty description |
|
|
45
|
+
| name-mismatch | MEDIUM | Name differs from package directory name |
|
|
46
|
+
| referenced-file | HIGH | Referenced path does not exist |
|
|
47
|
+
| script-not-executable | MEDIUM | Referenced scripts/ file has neither any executable bit nor a shebang |
|
|
48
|
+
| duplicate-skill-name | MEDIUM | Same nonempty name appears in multiple packages; each package is flagged |
|
|
49
|
+
| empty-body | MEDIUM | Stripped body has fewer than 50 characters |
|
|
50
|
+
|
|
51
|
+
Name and description are HIGH by design because they make a skill loadable
|
|
52
|
+
and discoverable. Name matching is a convention, not universal across hosts.
|
|
53
|
+
|
|
54
|
+
Path checks inspect the body after frontmatter, using backticked or bare
|
|
55
|
+
tokens containing `/` and ending in `.py`, `.js`, `.ts`, `.md`, `.json`,
|
|
56
|
+
`.yaml`, `.yml`, `.toml`, `.sh`, `.go`, `.rs`, or `.txt`, plus references
|
|
57
|
+
under `scripts/`, `references/`, `examples/`, `tests/`, `docs/`, and `bin/`.
|
|
58
|
+
URLs beginning http:// or https://, absolute paths, and tokens containing `*`
|
|
59
|
+
are skipped. Paths resolve against the skill directory first, then the audit
|
|
60
|
+
root. A trailing slash requires a directory. Shebang detection checks the
|
|
61
|
+
first two bytes for `#!`; permission checks inspect mode bits.
|
|
62
|
+
|
|
63
|
+
Findings include skill directory relative to the root, file, 1-based line,
|
|
64
|
+
kind, claim, severity, and detail. Metadata diagnostics use the opening lines;
|
|
65
|
+
body path diagnostics use their actual line. Findings deduplicate by
|
|
66
|
+
(skill, line, kind, claim). At most 500 are displayed, ordered by skill then
|
|
67
|
+
line (then kind and claim); truncation and omitted count are explicit.
|
|
68
|
+
Totals and verdict include all unignored findings, including omitted ones.
|
|
69
|
+
Human and JSON reports contain the same counts, per-kind and per-severity
|
|
70
|
+
totals, ignored count, parse-error count, and verdict.
|
|
71
|
+
|
|
72
|
+
## Exit codes
|
|
73
|
+
|
|
74
|
+
| Code | Verdict | Meaning |
|
|
75
|
+
| --- | --- | --- |
|
|
76
|
+
| 0 | PASS | No unignored findings or parse errors |
|
|
77
|
+
| 1 | WARN | Findings exist, all below the threshold |
|
|
78
|
+
| 2 | FAIL | Any finding reaches the threshold, or any parse error |
|
|
79
|
+
|
|
80
|
+
Default threshold is HIGH. With `--fail-on medium`, every finding fails.
|
|
81
|
+
An invalid audit path also produces exit 2. CLI argument errors use argparse's
|
|
82
|
+
exit 2. The warn-only case uses exit 1 even though no finding reaches the
|
|
83
|
+
threshold.
|
|
84
|
+
|
|
85
|
+
## Examples
|
|
86
|
+
|
|
87
|
+
All included examples are synthetic and need no credentials or services.
|
|
88
|
+
|
|
89
|
+
```sh
|
|
90
|
+
skill-drift audit examples/sample-skills/good-skill
|
|
91
|
+
# PASS: 1 skills found; no findings; exit 0
|
|
92
|
+
skill-drift audit examples/sample-skills/broken-skill
|
|
93
|
+
# FAIL: 1 skills found; exit 2
|
|
94
|
+
# HIGH missing-description: description
|
|
95
|
+
# MEDIUM name-mismatch: outdated-skill
|
|
96
|
+
# HIGH referenced-file: scripts/renamed.sh
|
|
97
|
+
# HIGH referenced-file: references/moved.md
|
|
98
|
+
skill-drift audit examples/sample-skills/broken-skill --json
|
|
99
|
+
# Valid JSON: HIGH=3, MEDIUM=1, verdict=FAIL, exit_code=2
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
The good package references an executable `scripts/check.sh`. The broken
|
|
103
|
+
package demonstrates a missing description, mismatched name, missing script,
|
|
104
|
+
and missing reference. The actual report also includes locations and details.
|
|
105
|
+
|
|
106
|
+
## Limitations
|
|
107
|
+
|
|
108
|
+
Checks are heuristic pattern matching, not semantic judgement. Paraphrased
|
|
109
|
+
claims are not checked. Commands are never executed; only their path tokens
|
|
110
|
+
are checked. Paths with spaces and unusual punctuation may be missed or
|
|
111
|
+
misidentified. Existence says nothing about correctness, safety, importability,
|
|
112
|
+
or successful execution. External targets and glob expansions are not checked.
|
|
113
|
+
The YAML subset rejects complex YAML instead of guessing. Filesystem changes
|
|
114
|
+
during a run can affect results; this is a snapshot-style audit, not a monitor.
|
|
115
|
+
|
|
116
|
+
## Differentiation
|
|
117
|
+
|
|
118
|
+
- **instruction-drift** audits instruction files (AGENTS.md, CLAUDE.md,
|
|
119
|
+
MUSE.md, CONTRIBUTING.md) against a repository. This audits skill packages:
|
|
120
|
+
SKILL.md, frontmatter, scripts, and references against the disk.
|
|
121
|
+
- **skill-tax** audits what a SKILL.md costs in tokens. This audits whether
|
|
122
|
+
the skill still works through its checkable package claims.
|
|
123
|
+
- **skill-pin-verify** audits pinning/signatures. This audits the truth of
|
|
124
|
+
package contents through offline heuristics.
|
|
125
|
+
- **frontmatter-guard** audits plugin.json manifests. This audits SKILL.md
|
|
126
|
+
frontmatter.
|
|
127
|
+
- **Heavy runtimes** require more infrastructure. This is an offline stdlib
|
|
128
|
+
CLI with no daemon, no LLM, and no network.
|
|
129
|
+
|
|
130
|
+
## Development
|
|
131
|
+
|
|
132
|
+
```sh
|
|
133
|
+
PYTHONPATH=src python -m unittest discover -s tests -v
|
|
134
|
+
python -m build
|
|
135
|
+
python -m venv /tmp/skill-drift-smoke
|
|
136
|
+
/tmp/skill-drift-smoke/bin/pip install dist/skill_drift-0.1.0-py3-none-any.whl
|
|
137
|
+
/tmp/skill-drift-smoke/bin/skill-drift audit examples/sample-skills/good-skill
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
Tests use unittest, temporary synthetic fixtures, and no network. Setuptools
|
|
141
|
+
and wheel are build-time dependencies; `build` is the development frontend.
|
|
142
|
+
Build artifacts are local only and should be removed before sharing a tree.
|
|
143
|
+
|
|
144
|
+
MIT licensed. See LICENSE.
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=61", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "skill-drift"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Offline heuristic auditing of skill package claims"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.9"
|
|
11
|
+
license = {text = "MIT"}
|
|
12
|
+
authors = [{name = "skill-drift contributors"}]
|
|
13
|
+
dependencies = []
|
|
14
|
+
|
|
15
|
+
[project.scripts]
|
|
16
|
+
skill-drift = "skill_drift.cli:main"
|
|
17
|
+
|
|
18
|
+
[tool.setuptools.packages.find]
|
|
19
|
+
where = ["src"]
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
"""Offline heuristic checks; never executes package code."""
|
|
2
|
+
from collections import Counter
|
|
3
|
+
from dataclasses import asdict, dataclass
|
|
4
|
+
from fnmatch import fnmatchcase
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
import re
|
|
7
|
+
from .skills import discover, read_skill
|
|
8
|
+
|
|
9
|
+
@dataclass
|
|
10
|
+
class Finding:
|
|
11
|
+
skill: str
|
|
12
|
+
file: str
|
|
13
|
+
line: int
|
|
14
|
+
kind: str
|
|
15
|
+
claim: str
|
|
16
|
+
severity: str
|
|
17
|
+
detail: str
|
|
18
|
+
|
|
19
|
+
TOKEN = re.compile(r'`([^`\n]+)`|([^\s`<>"\']+)')
|
|
20
|
+
EXT = re.compile(r'\.(py|js|ts|md|json|yaml|yml|toml|sh|go|rs|txt)$')
|
|
21
|
+
DIR = re.compile(r'^(?:\./)?(?:scripts|references|examples|tests|docs|bin)/')
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def claims(line):
|
|
25
|
+
for match in TOKEN.finditer(line):
|
|
26
|
+
token = (match[1] or match[2]).strip('()[]{}.,;:!?')
|
|
27
|
+
if token.startswith(('http://', 'https://', '/', '~/', '\\')) or '*' in token or re.match(r'^[A-Za-z]:[/\\]', token):
|
|
28
|
+
continue
|
|
29
|
+
if '/' in token and (EXT.search(token) or DIR.match(token)):
|
|
30
|
+
yield token
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def audit(root, fail_on='high', ignore=(), limit=500):
|
|
34
|
+
root = Path(root).resolve()
|
|
35
|
+
packages = [read_skill(p) for p in discover(root)]
|
|
36
|
+
findings = []
|
|
37
|
+
errors = []
|
|
38
|
+
names = Counter(s.metadata.get('name') for s in packages if s.metadata.get('name'))
|
|
39
|
+
ignored = 0
|
|
40
|
+
seen = set()
|
|
41
|
+
for skill in packages:
|
|
42
|
+
relative = skill.directory.relative_to(root).as_posix()
|
|
43
|
+
if skill.error:
|
|
44
|
+
errors.append({'skill': relative, 'file': 'SKILL.md', 'line': 1, 'detail': skill.error})
|
|
45
|
+
continue
|
|
46
|
+
def add(line, kind, claim, severity, detail):
|
|
47
|
+
nonlocal ignored
|
|
48
|
+
key = (relative, line, kind, claim)
|
|
49
|
+
if key in seen:
|
|
50
|
+
return
|
|
51
|
+
seen.add(key)
|
|
52
|
+
if any(fnmatchcase(claim, pattern) for pattern in ignore):
|
|
53
|
+
ignored += 1
|
|
54
|
+
return
|
|
55
|
+
findings.append(Finding(relative, 'SKILL.md', line, kind, claim, severity, detail))
|
|
56
|
+
if skill.missing_frontmatter:
|
|
57
|
+
add(1, 'missing-frontmatter', 'frontmatter', 'HIGH', 'Expected opening --- frontmatter line')
|
|
58
|
+
else:
|
|
59
|
+
for key in ('name', 'description'):
|
|
60
|
+
if not skill.metadata.get(key, '').strip():
|
|
61
|
+
add(1, 'missing-' + key, key, 'HIGH', 'Required nonempty frontmatter field')
|
|
62
|
+
name = skill.metadata.get('name')
|
|
63
|
+
if name and name != skill.directory.name:
|
|
64
|
+
add(2, 'name-mismatch', name, 'MEDIUM', 'Directory name is ' + skill.directory.name)
|
|
65
|
+
if name and names[name] > 1:
|
|
66
|
+
add(2, 'duplicate-skill-name', name, 'MEDIUM', 'Name occurs in multiple skill packages')
|
|
67
|
+
if len(skill.body.strip()) < 50:
|
|
68
|
+
add(skill.body_line, 'empty-body', 'body', 'MEDIUM', 'Body is shorter than 50 characters')
|
|
69
|
+
for number, line in enumerate(skill.body.splitlines(), skill.body_line):
|
|
70
|
+
for claim in claims(line):
|
|
71
|
+
local = skill.directory / claim
|
|
72
|
+
target = local if local.exists() else root / claim
|
|
73
|
+
exists = target.is_dir() if claim.endswith('/') else target.exists()
|
|
74
|
+
if not exists:
|
|
75
|
+
add(number, 'referenced-file', claim, 'HIGH', 'Referenced path does not exist')
|
|
76
|
+
elif re.match(r'^(?:\./)?scripts/', claim) and target.is_file():
|
|
77
|
+
try:
|
|
78
|
+
shebang = target.open('rb')
|
|
79
|
+
with shebang:
|
|
80
|
+
starts = shebang.read(2) == b'#!'
|
|
81
|
+
if not target.stat().st_mode & 0o111 and not starts:
|
|
82
|
+
add(number, 'script-not-executable', claim, 'MEDIUM', 'No executable bit and no shebang')
|
|
83
|
+
except OSError as error:
|
|
84
|
+
errors.append({'skill': relative, 'file': claim, 'line': number, 'detail': str(error)})
|
|
85
|
+
findings.sort(key=lambda f: (f.skill, f.line, f.kind, f.claim))
|
|
86
|
+
# Verdict considers all findings, including those beyond the display cap.
|
|
87
|
+
failed = bool(errors) or any(f.severity == 'HIGH' or fail_on == 'medium' for f in findings)
|
|
88
|
+
code = 2 if failed else (1 if findings else 0)
|
|
89
|
+
return {'skills_found': len(packages), 'totals': {'kind': dict(sorted(Counter(f.kind for f in findings).items())), 'severity': dict(sorted(Counter(f.severity for f in findings).items()))}, 'findings': [asdict(f) for f in findings[:limit]], 'ignored_count': ignored, 'parse_error_count': len(errors), 'parse_errors': errors, 'truncated': len(findings) > limit, 'omitted_count': max(0, len(findings)-limit), 'verdict': ('FAIL' if code == 2 else 'WARN' if code == 1 else 'PASS'), 'exit_code': code, 'note': 'no skills found' if not packages else ''}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
"""Command line interface."""
|
|
2
|
+
import argparse
|
|
3
|
+
import json
|
|
4
|
+
from . import __version__
|
|
5
|
+
from .audit import audit
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def main(argv=None):
|
|
9
|
+
parser = argparse.ArgumentParser(prog='skill-drift', description='Audit skill package claims offline')
|
|
10
|
+
parser.add_argument('--version', action='version', version='%(prog)s ' + __version__)
|
|
11
|
+
commands = parser.add_subparsers(dest='command', required=True)
|
|
12
|
+
command = commands.add_parser('audit')
|
|
13
|
+
command.add_argument('path', nargs='?', default='.')
|
|
14
|
+
command.add_argument('--json', action='store_true')
|
|
15
|
+
command.add_argument('--fail-on', choices=('high', 'medium'), default='high')
|
|
16
|
+
command.add_argument('--ignore', action='append', default=[], metavar='G')
|
|
17
|
+
args = parser.parse_args(argv)
|
|
18
|
+
try:
|
|
19
|
+
report = audit(args.path, args.fail_on, args.ignore)
|
|
20
|
+
except (ValueError, OSError) as error:
|
|
21
|
+
report = {'skills_found': 0, 'totals': {'kind': {}, 'severity': {}}, 'findings': [], 'ignored_count': 0, 'parse_error_count': 1, 'parse_errors': [{'skill': '.', 'file': 'SKILL.md', 'line': 1, 'detail': str(error)}], 'truncated': False, 'omitted_count': 0, 'verdict': 'FAIL', 'exit_code': 2, 'note': ''}
|
|
22
|
+
if args.json:
|
|
23
|
+
print(json.dumps(report, sort_keys=True))
|
|
24
|
+
else:
|
|
25
|
+
print('%s: %s skills found' % (report['verdict'], report['skills_found']))
|
|
26
|
+
if report['note']:
|
|
27
|
+
print(report['note'])
|
|
28
|
+
for finding in report['findings']:
|
|
29
|
+
print('{skill}:{file}:{line}: {severity} {kind}: {claim} — {detail}'.format(**finding))
|
|
30
|
+
for error in report['parse_errors']:
|
|
31
|
+
print('{skill}:{file}:{line}: parse error: {detail}'.format(**error))
|
|
32
|
+
print('Totals: ' + json.dumps(report['totals'], sort_keys=True))
|
|
33
|
+
print('Ignored: %s; parse errors: %s; truncated: %s (omitted %s)' % (report['ignored_count'], report['parse_error_count'], report['truncated'], report['omitted_count']))
|
|
34
|
+
return report['exit_code']
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
"""Deterministic discovery and deliberately small frontmatter parser."""
|
|
2
|
+
from dataclasses import dataclass
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
import os
|
|
5
|
+
import re
|
|
6
|
+
|
|
7
|
+
SKIP = {'.git', 'node_modules', 'dist', '__pycache__', '.venv', 'venv'}
|
|
8
|
+
|
|
9
|
+
@dataclass
|
|
10
|
+
class Skill:
|
|
11
|
+
directory: Path
|
|
12
|
+
metadata: dict
|
|
13
|
+
body: str
|
|
14
|
+
body_line: int
|
|
15
|
+
error: str = ''
|
|
16
|
+
missing_frontmatter: bool = False
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def discover(root):
|
|
20
|
+
root = Path(root)
|
|
21
|
+
if not root.is_dir():
|
|
22
|
+
raise ValueError('audit path must be an existing directory')
|
|
23
|
+
found = []
|
|
24
|
+
def walk_error(error):
|
|
25
|
+
raise ValueError(str(error))
|
|
26
|
+
for directory, dirs, files in os.walk(root, onerror=walk_error):
|
|
27
|
+
dirs[:] = sorted(d for d in dirs if d not in SKIP)
|
|
28
|
+
if 'SKILL.md' in files:
|
|
29
|
+
found.append(Path(directory))
|
|
30
|
+
return sorted(found, key=lambda p: p.relative_to(root).as_posix())
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def scalar(value):
|
|
34
|
+
value = value.strip()
|
|
35
|
+
if not value:
|
|
36
|
+
return ''
|
|
37
|
+
if value[0] in "\"'":
|
|
38
|
+
if len(value) < 2 or value[-1] != value[0]:
|
|
39
|
+
raise ValueError('unterminated quoted scalar')
|
|
40
|
+
return value[1:-1]
|
|
41
|
+
if re.search(r'(^|\s)[&*!]|:\s|^[>|\[\]{}]|^[-?]\s', value):
|
|
42
|
+
raise ValueError('unsupported complex YAML scalar')
|
|
43
|
+
return value
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def read_skill(directory):
|
|
47
|
+
directory = Path(directory)
|
|
48
|
+
try:
|
|
49
|
+
lines = (directory / 'SKILL.md').read_text(encoding='utf-8').splitlines()
|
|
50
|
+
if not lines or lines[0] != '---':
|
|
51
|
+
return Skill(directory, {}, '\n'.join(lines), 1, missing_frontmatter=True)
|
|
52
|
+
try:
|
|
53
|
+
end = lines.index('---', 1)
|
|
54
|
+
except ValueError:
|
|
55
|
+
raise ValueError('frontmatter has no closing --- line')
|
|
56
|
+
metadata = {}
|
|
57
|
+
current = None
|
|
58
|
+
for number, line in enumerate(lines[1:end], 2):
|
|
59
|
+
if not line.strip() or line.lstrip().startswith('#'):
|
|
60
|
+
continue
|
|
61
|
+
item = re.fullmatch(r'\s+-\s+(.+)', line)
|
|
62
|
+
if item and current:
|
|
63
|
+
if metadata[current] == '':
|
|
64
|
+
metadata[current] = []
|
|
65
|
+
if not isinstance(metadata[current], list):
|
|
66
|
+
raise ValueError('list under nonempty scalar')
|
|
67
|
+
metadata[current].append(scalar(item[1]))
|
|
68
|
+
continue
|
|
69
|
+
match = re.fullmatch(r'([A-Za-z_][\w-]*):\s*(.*)', line)
|
|
70
|
+
if not match:
|
|
71
|
+
raise ValueError('unsupported YAML at line %s' % number)
|
|
72
|
+
current = match[1]
|
|
73
|
+
if current in metadata:
|
|
74
|
+
raise ValueError('duplicate frontmatter key: ' + current)
|
|
75
|
+
metadata[current] = scalar(match[2])
|
|
76
|
+
for key in ('name', 'description'):
|
|
77
|
+
if key in metadata and not isinstance(metadata[key], str):
|
|
78
|
+
raise ValueError(key + ' must be a scalar')
|
|
79
|
+
return Skill(directory, metadata, '\n'.join(lines[end+1:]), end+2)
|
|
80
|
+
except (OSError, UnicodeError, ValueError) as error:
|
|
81
|
+
return Skill(directory, {}, '', 1, error=str(error))
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: skill-drift
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Offline heuristic auditing of skill package claims
|
|
5
|
+
Author: skill-drift contributors
|
|
6
|
+
License: MIT
|
|
7
|
+
Requires-Python: >=3.9
|
|
8
|
+
Description-Content-Type: text/markdown
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Dynamic: license-file
|
|
11
|
+
|
|
12
|
+
# skill-drift
|
|
13
|
+
|
|
14
|
+
`skill-drift` 0.1.0 checks whether an agent skill package's stated paths and
|
|
15
|
+
basic metadata still match the disk. It is deterministic, offline, and uses
|
|
16
|
+
only the Python standard library at runtime: no daemon, LLM, network, or keys.
|
|
17
|
+
A finding means **check this claim**.
|
|
18
|
+
|
|
19
|
+
Requires Python 3.9+. Install with `pip install .`, or install a built wheel.
|
|
20
|
+
|
|
21
|
+
```sh
|
|
22
|
+
skill-drift audit [path] [--json] [--fail-on high|medium] [--ignore G]
|
|
23
|
+
skill-drift --version
|
|
24
|
+
skill-drift --help
|
|
25
|
+
python -m skill_drift audit .
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
Path defaults to the current directory. Repeat `--ignore` for multiple glob
|
|
29
|
+
patterns; patterns match the claim text, apply to all checks, and excluded
|
|
30
|
+
unique findings are counted as ignored.
|
|
31
|
+
|
|
32
|
+
## Discovery and parsing
|
|
33
|
+
|
|
34
|
+
A package is any directory containing a file named exactly `SKILL.md`.
|
|
35
|
+
Discovery recursively includes the supplied directory itself and hidden
|
|
36
|
+
folders such as `.claude/skills` and `.cursor/skills`. It skips `.git`,
|
|
37
|
+
`node_modules`, `dist`, `__pycache__`, `.venv`, and `venv`; directory symlinks
|
|
38
|
+
are not traversed. No packages gives PASS, exit 0, and `no skills found`.
|
|
39
|
+
Unreadable or invalid UTF-8 SKILL.md files produce parse errors.
|
|
40
|
+
|
|
41
|
+
The file must begin with an exact `---` line and close its frontmatter with
|
|
42
|
+
another `---` line. Supported YAML is a subset: `key: value`, quoted or
|
|
43
|
+
unquoted scalar values, and block lists under a key. Blank lines and full-line
|
|
44
|
+
comments are accepted. Nested maps, anchors, tags, flow collections, multiline
|
|
45
|
+
scalars, duplicate keys, and malformed blocks are parse errors. Name and
|
|
46
|
+
description must be scalars. Quoted values are treated literally, without
|
|
47
|
+
YAML escape processing; inline comments are not interpreted.
|
|
48
|
+
|
|
49
|
+
## Checks and severities
|
|
50
|
+
|
|
51
|
+
| Kind | Severity | Trigger |
|
|
52
|
+
| --- | --- | --- |
|
|
53
|
+
| missing-frontmatter | HIGH | No opening frontmatter delimiter |
|
|
54
|
+
| missing-name | HIGH | Missing or empty name |
|
|
55
|
+
| missing-description | HIGH | Missing or empty description |
|
|
56
|
+
| name-mismatch | MEDIUM | Name differs from package directory name |
|
|
57
|
+
| referenced-file | HIGH | Referenced path does not exist |
|
|
58
|
+
| script-not-executable | MEDIUM | Referenced scripts/ file has neither any executable bit nor a shebang |
|
|
59
|
+
| duplicate-skill-name | MEDIUM | Same nonempty name appears in multiple packages; each package is flagged |
|
|
60
|
+
| empty-body | MEDIUM | Stripped body has fewer than 50 characters |
|
|
61
|
+
|
|
62
|
+
Name and description are HIGH by design because they make a skill loadable
|
|
63
|
+
and discoverable. Name matching is a convention, not universal across hosts.
|
|
64
|
+
|
|
65
|
+
Path checks inspect the body after frontmatter, using backticked or bare
|
|
66
|
+
tokens containing `/` and ending in `.py`, `.js`, `.ts`, `.md`, `.json`,
|
|
67
|
+
`.yaml`, `.yml`, `.toml`, `.sh`, `.go`, `.rs`, or `.txt`, plus references
|
|
68
|
+
under `scripts/`, `references/`, `examples/`, `tests/`, `docs/`, and `bin/`.
|
|
69
|
+
URLs beginning http:// or https://, absolute paths, and tokens containing `*`
|
|
70
|
+
are skipped. Paths resolve against the skill directory first, then the audit
|
|
71
|
+
root. A trailing slash requires a directory. Shebang detection checks the
|
|
72
|
+
first two bytes for `#!`; permission checks inspect mode bits.
|
|
73
|
+
|
|
74
|
+
Findings include skill directory relative to the root, file, 1-based line,
|
|
75
|
+
kind, claim, severity, and detail. Metadata diagnostics use the opening lines;
|
|
76
|
+
body path diagnostics use their actual line. Findings deduplicate by
|
|
77
|
+
(skill, line, kind, claim). At most 500 are displayed, ordered by skill then
|
|
78
|
+
line (then kind and claim); truncation and omitted count are explicit.
|
|
79
|
+
Totals and verdict include all unignored findings, including omitted ones.
|
|
80
|
+
Human and JSON reports contain the same counts, per-kind and per-severity
|
|
81
|
+
totals, ignored count, parse-error count, and verdict.
|
|
82
|
+
|
|
83
|
+
## Exit codes
|
|
84
|
+
|
|
85
|
+
| Code | Verdict | Meaning |
|
|
86
|
+
| --- | --- | --- |
|
|
87
|
+
| 0 | PASS | No unignored findings or parse errors |
|
|
88
|
+
| 1 | WARN | Findings exist, all below the threshold |
|
|
89
|
+
| 2 | FAIL | Any finding reaches the threshold, or any parse error |
|
|
90
|
+
|
|
91
|
+
Default threshold is HIGH. With `--fail-on medium`, every finding fails.
|
|
92
|
+
An invalid audit path also produces exit 2. CLI argument errors use argparse's
|
|
93
|
+
exit 2. The warn-only case uses exit 1 even though no finding reaches the
|
|
94
|
+
threshold.
|
|
95
|
+
|
|
96
|
+
## Examples
|
|
97
|
+
|
|
98
|
+
All included examples are synthetic and need no credentials or services.
|
|
99
|
+
|
|
100
|
+
```sh
|
|
101
|
+
skill-drift audit examples/sample-skills/good-skill
|
|
102
|
+
# PASS: 1 skills found; no findings; exit 0
|
|
103
|
+
skill-drift audit examples/sample-skills/broken-skill
|
|
104
|
+
# FAIL: 1 skills found; exit 2
|
|
105
|
+
# HIGH missing-description: description
|
|
106
|
+
# MEDIUM name-mismatch: outdated-skill
|
|
107
|
+
# HIGH referenced-file: scripts/renamed.sh
|
|
108
|
+
# HIGH referenced-file: references/moved.md
|
|
109
|
+
skill-drift audit examples/sample-skills/broken-skill --json
|
|
110
|
+
# Valid JSON: HIGH=3, MEDIUM=1, verdict=FAIL, exit_code=2
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
The good package references an executable `scripts/check.sh`. The broken
|
|
114
|
+
package demonstrates a missing description, mismatched name, missing script,
|
|
115
|
+
and missing reference. The actual report also includes locations and details.
|
|
116
|
+
|
|
117
|
+
## Limitations
|
|
118
|
+
|
|
119
|
+
Checks are heuristic pattern matching, not semantic judgement. Paraphrased
|
|
120
|
+
claims are not checked. Commands are never executed; only their path tokens
|
|
121
|
+
are checked. Paths with spaces and unusual punctuation may be missed or
|
|
122
|
+
misidentified. Existence says nothing about correctness, safety, importability,
|
|
123
|
+
or successful execution. External targets and glob expansions are not checked.
|
|
124
|
+
The YAML subset rejects complex YAML instead of guessing. Filesystem changes
|
|
125
|
+
during a run can affect results; this is a snapshot-style audit, not a monitor.
|
|
126
|
+
|
|
127
|
+
## Differentiation
|
|
128
|
+
|
|
129
|
+
- **instruction-drift** audits instruction files (AGENTS.md, CLAUDE.md,
|
|
130
|
+
MUSE.md, CONTRIBUTING.md) against a repository. This audits skill packages:
|
|
131
|
+
SKILL.md, frontmatter, scripts, and references against the disk.
|
|
132
|
+
- **skill-tax** audits what a SKILL.md costs in tokens. This audits whether
|
|
133
|
+
the skill still works through its checkable package claims.
|
|
134
|
+
- **skill-pin-verify** audits pinning/signatures. This audits the truth of
|
|
135
|
+
package contents through offline heuristics.
|
|
136
|
+
- **frontmatter-guard** audits plugin.json manifests. This audits SKILL.md
|
|
137
|
+
frontmatter.
|
|
138
|
+
- **Heavy runtimes** require more infrastructure. This is an offline stdlib
|
|
139
|
+
CLI with no daemon, no LLM, and no network.
|
|
140
|
+
|
|
141
|
+
## Development
|
|
142
|
+
|
|
143
|
+
```sh
|
|
144
|
+
PYTHONPATH=src python -m unittest discover -s tests -v
|
|
145
|
+
python -m build
|
|
146
|
+
python -m venv /tmp/skill-drift-smoke
|
|
147
|
+
/tmp/skill-drift-smoke/bin/pip install dist/skill_drift-0.1.0-py3-none-any.whl
|
|
148
|
+
/tmp/skill-drift-smoke/bin/skill-drift audit examples/sample-skills/good-skill
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
Tests use unittest, temporary synthetic fixtures, and no network. Setuptools
|
|
152
|
+
and wheel are build-time dependencies; `build` is the development frontend.
|
|
153
|
+
Build artifacts are local only and should be removed before sharing a tree.
|
|
154
|
+
|
|
155
|
+
MIT licensed. See LICENSE.
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
README.md
|
|
3
|
+
pyproject.toml
|
|
4
|
+
src/skill_drift/__init__.py
|
|
5
|
+
src/skill_drift/__main__.py
|
|
6
|
+
src/skill_drift/audit.py
|
|
7
|
+
src/skill_drift/cli.py
|
|
8
|
+
src/skill_drift/skills.py
|
|
9
|
+
src/skill_drift.egg-info/PKG-INFO
|
|
10
|
+
src/skill_drift.egg-info/SOURCES.txt
|
|
11
|
+
src/skill_drift.egg-info/dependency_links.txt
|
|
12
|
+
src/skill_drift.egg-info/entry_points.txt
|
|
13
|
+
src/skill_drift.egg-info/top_level.txt
|
|
14
|
+
tests/test_audit.py
|
|
15
|
+
tests/test_cli.py
|
|
16
|
+
tests/test_discovery.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
skill_drift
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
import tempfile
|
|
2
|
+
import unittest
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from skill_drift.audit import audit
|
|
5
|
+
|
|
6
|
+
BODY = 'Follow this synthetic procedure carefully using local fixture data only. '
|
|
7
|
+
|
|
8
|
+
class AuditTests(unittest.TestCase):
|
|
9
|
+
def setUp(self):
|
|
10
|
+
self.tmp = tempfile.TemporaryDirectory(); self.addCleanup(self.tmp.cleanup)
|
|
11
|
+
self.root = Path(self.tmp.name)
|
|
12
|
+
def skill(self, name='demo', front=None, body=BODY):
|
|
13
|
+
p = self.root/name; p.mkdir(parents=True, exist_ok=True)
|
|
14
|
+
(p/'SKILL.md').write_text('---\n'+(front if front is not None else 'name: '+name+'\ndescription: synthetic')+'\n---\n'+body)
|
|
15
|
+
return p
|
|
16
|
+
def report(self, **kw): return audit(self.root, **kw)
|
|
17
|
+
def kinds(self): return self.report()['totals']['kind']
|
|
18
|
+
def test_good(self):
|
|
19
|
+
self.skill(); self.assertEqual(self.report()['exit_code'], 0)
|
|
20
|
+
def test_missing_frontmatter(self):
|
|
21
|
+
p=self.skill(); (p/'SKILL.md').write_text(BODY)
|
|
22
|
+
self.assertEqual(self.kinds(), {'missing-frontmatter':1}); self.assertEqual(self.report()['exit_code'],2)
|
|
23
|
+
def test_missing_name(self):
|
|
24
|
+
self.skill(front='description: synthetic'); self.assertEqual(self.kinds(), {'missing-name':1})
|
|
25
|
+
def test_empty_name(self):
|
|
26
|
+
self.skill(front='name: ""\ndescription: synthetic'); self.assertIn('missing-name',self.kinds())
|
|
27
|
+
def test_missing_description(self):
|
|
28
|
+
self.skill(front='name: demo'); self.assertEqual(self.report()['totals']['severity'], {'HIGH':1})
|
|
29
|
+
def test_empty_description(self):
|
|
30
|
+
self.skill(front='name: demo\ndescription:'); self.assertIn('missing-description',self.kinds())
|
|
31
|
+
def test_mismatch_warn(self):
|
|
32
|
+
self.skill(front='name: other\ndescription: synthetic'); self.assertEqual(self.report()['exit_code'],1)
|
|
33
|
+
def test_mismatch_fail(self):
|
|
34
|
+
self.skill(front='name: other\ndescription: synthetic'); self.assertEqual(self.report(fail_on='medium')['exit_code'],2)
|
|
35
|
+
def test_missing_reference(self):
|
|
36
|
+
self.skill(body=BODY+'`references/missing.md`'); self.assertEqual(self.kinds(), {'referenced-file':1})
|
|
37
|
+
def test_existing_reference(self):
|
|
38
|
+
p=self.skill(body=BODY+'references/a.md'); (p/'references').mkdir(); (p/'references/a.md').touch()
|
|
39
|
+
self.assertEqual(self.report()['exit_code'],0)
|
|
40
|
+
def test_script_no_exec(self):
|
|
41
|
+
p=self.skill(body=BODY+'scripts/a.sh'); (p/'scripts').mkdir(); f=p/'scripts/a.sh'; f.write_text('echo synthetic'); f.chmod(0o644)
|
|
42
|
+
self.assertEqual(self.kinds(), {'script-not-executable':1})
|
|
43
|
+
def test_script_shebang(self):
|
|
44
|
+
p=self.skill(body=BODY+'scripts/a.sh'); (p/'scripts').mkdir(); f=p/'scripts/a.sh'; f.write_text('#!/bin/sh\n'); f.chmod(0o644)
|
|
45
|
+
self.assertEqual(self.report()['exit_code'],0)
|
|
46
|
+
def test_script_exec(self):
|
|
47
|
+
p=self.skill(body=BODY+'scripts/a.sh'); (p/'scripts').mkdir(); f=p/'scripts/a.sh'; f.touch(); f.chmod(0o755)
|
|
48
|
+
self.assertEqual(self.report()['exit_code'],0)
|
|
49
|
+
def test_skipped_tokens(self):
|
|
50
|
+
self.skill(body=BODY+'`https://example.invalid/a.md` http://example.invalid/b.py /abs/a.md `scripts/*.sh`')
|
|
51
|
+
self.assertEqual(self.report()['exit_code'],0)
|
|
52
|
+
def test_root_fallback(self):
|
|
53
|
+
self.skill(body=BODY+'docs/a.md'); (self.root/'docs').mkdir(); (self.root/'docs/a.md').touch()
|
|
54
|
+
self.assertEqual(self.report()['exit_code'],0)
|
|
55
|
+
def test_local_precedence(self):
|
|
56
|
+
p=self.skill(body=BODY+'scripts/a.sh'); (p/'scripts').mkdir(); f=p/'scripts/a.sh'; f.write_text('#!/bin/sh'); f.chmod(0o644)
|
|
57
|
+
(self.root/'scripts').mkdir(); f=self.root/'scripts/a.sh'; f.touch(); f.chmod(0o644)
|
|
58
|
+
self.assertEqual(self.report()['exit_code'],0)
|
|
59
|
+
def test_duplicate_names(self):
|
|
60
|
+
self.skill(); self.skill('other',front='name: demo\ndescription: synthetic')
|
|
61
|
+
self.assertEqual(self.kinds()['duplicate-skill-name'],2)
|
|
62
|
+
def test_short_body(self):
|
|
63
|
+
self.skill(body='short'); self.assertEqual(self.kinds(), {'empty-body':1})
|
|
64
|
+
def test_body_boundary(self):
|
|
65
|
+
self.skill(body='x'*50); self.assertEqual(self.report()['exit_code'],0)
|
|
66
|
+
def test_ignore(self):
|
|
67
|
+
self.skill(body=BODY+'scripts/missing.sh references/missing.md')
|
|
68
|
+
r=self.report(ignore=['scripts/*','references/*']); self.assertEqual(r['ignored_count'],2); self.assertEqual(r['exit_code'],0)
|
|
69
|
+
def test_dedup(self):
|
|
70
|
+
self.skill(body=BODY+'scripts/a.sh `scripts/a.sh`'); self.assertEqual(self.kinds()['referenced-file'],1)
|
|
71
|
+
def test_distinct_lines(self):
|
|
72
|
+
self.skill(body=BODY+'scripts/a.sh\nscripts/a.sh'); self.assertEqual(self.kinds()['referenced-file'],2)
|
|
73
|
+
def test_directory_claim(self):
|
|
74
|
+
p=self.skill(body=BODY+'references/'); self.assertEqual(self.report()['exit_code'],2)
|
|
75
|
+
(p/'references').mkdir(); self.assertEqual(self.report()['exit_code'],0)
|
|
76
|
+
def test_directory_cannot_be_file(self):
|
|
77
|
+
p=self.skill(body=BODY+'docs/'); (p/'docs').touch(); self.assertEqual(self.report()['exit_code'],2)
|
|
78
|
+
def test_realistic(self):
|
|
79
|
+
self.skill(front='name: old\ndescription: synthetic',body=BODY+'`scripts/old.py` `references/old.md`')
|
|
80
|
+
self.assertEqual(self.report()['totals']['severity'], {'HIGH':2,'MEDIUM':1}); self.assertEqual(self.report()['exit_code'],2)
|
|
81
|
+
def test_cap(self):
|
|
82
|
+
self.skill(body=BODY+'\n'+'\n'.join('docs/%s.md'%i for i in range(510)))
|
|
83
|
+
r=self.report(); self.assertEqual(len(r['findings']),500); self.assertEqual(r['omitted_count'],10); self.assertTrue(r['truncated']); self.assertEqual(r['totals']['kind']['referenced-file'],510)
|
|
84
|
+
def test_order(self):
|
|
85
|
+
self.skill('z',body=BODY+'docs/z.md'); self.skill('a',body=BODY+'docs/a.md')
|
|
86
|
+
self.assertEqual([f['skill'] for f in self.report()['findings']], ['a','z'])
|
|
87
|
+
def test_body_line(self):
|
|
88
|
+
self.skill(body=BODY+'docs/a.md'); self.assertEqual(self.report()['findings'][0]['line'],5)
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import contextlib
|
|
2
|
+
import io
|
|
3
|
+
import json
|
|
4
|
+
import tempfile
|
|
5
|
+
import unittest
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from skill_drift.cli import main
|
|
8
|
+
|
|
9
|
+
class CliTests(unittest.TestCase):
|
|
10
|
+
def setUp(self):
|
|
11
|
+
self.tmp=tempfile.TemporaryDirectory(); self.addCleanup(self.tmp.cleanup); self.root=Path(self.tmp.name)
|
|
12
|
+
def run_cli(self,*args):
|
|
13
|
+
output=io.StringIO()
|
|
14
|
+
with contextlib.redirect_stdout(output): code=main(list(args))
|
|
15
|
+
return code,output.getvalue()
|
|
16
|
+
def write(self,front='name: demo\ndescription: synthetic',body='Synthetic instructions for a local fixture with sufficient detail.'):
|
|
17
|
+
p=self.root/'demo'; p.mkdir(exist_ok=True); (p/'SKILL.md').write_text('---\n'+front+'\n---\n'+body)
|
|
18
|
+
def test_no_skills(self):
|
|
19
|
+
code,out=self.run_cli('audit',str(self.root)); self.assertEqual(code,0); self.assertIn('no skills found',out)
|
|
20
|
+
def test_good(self):
|
|
21
|
+
self.write(); self.assertEqual(self.run_cli('audit',str(self.root))[0],0)
|
|
22
|
+
def test_high(self):
|
|
23
|
+
self.write(front='name: demo'); self.assertEqual(self.run_cli('audit',str(self.root))[0],2)
|
|
24
|
+
def test_warn(self):
|
|
25
|
+
self.write(front='name: old\ndescription: synthetic'); self.assertEqual(self.run_cli('audit',str(self.root))[0],1)
|
|
26
|
+
def test_medium_threshold(self):
|
|
27
|
+
self.write(front='name: old\ndescription: synthetic'); self.assertEqual(self.run_cli('audit',str(self.root),'--fail-on','medium')[0],2)
|
|
28
|
+
def test_json(self):
|
|
29
|
+
self.write(); code,out=self.run_cli('audit',str(self.root),'--json'); r=json.loads(out); self.assertEqual(code,r['exit_code']); self.assertEqual(r['verdict'],'PASS')
|
|
30
|
+
def test_parse_error(self):
|
|
31
|
+
self.write(front='name: demo\ndescription: |'); code,out=self.run_cli('audit',str(self.root),'--json'); self.assertEqual(code,2); self.assertEqual(json.loads(out)['parse_error_count'],1)
|
|
32
|
+
def test_missing_path(self):
|
|
33
|
+
code,out=self.run_cli('audit',str(self.root/'missing'),'--json'); self.assertEqual(code,2); self.assertEqual(json.loads(out)['parse_error_count'],1)
|
|
34
|
+
def test_repeat_ignore(self):
|
|
35
|
+
self.write(body='Synthetic instructions with adequate detail. scripts/a.sh docs/a.md')
|
|
36
|
+
code,out=self.run_cli('audit',str(self.root),'--ignore','scripts/*','--ignore','docs/*','--json'); self.assertEqual(code,0); self.assertEqual(json.loads(out)['ignored_count'],2)
|
|
37
|
+
def test_version(self):
|
|
38
|
+
out=io.StringIO()
|
|
39
|
+
with contextlib.redirect_stdout(out), self.assertRaises(SystemExit) as cm: main(['--version'])
|
|
40
|
+
self.assertEqual(cm.exception.code,0); self.assertIn('0.1.0',out.getvalue())
|
|
41
|
+
def test_help(self):
|
|
42
|
+
with contextlib.redirect_stdout(io.StringIO()), self.assertRaises(SystemExit) as cm: main(['--help'])
|
|
43
|
+
self.assertEqual(cm.exception.code,0)
|
|
44
|
+
def test_missing_frontmatter(self):
|
|
45
|
+
self.write(); (self.root/'demo/SKILL.md').write_text('Synthetic instructions with enough characters to pass body length.')
|
|
46
|
+
code,out=self.run_cli('audit',str(self.root)); self.assertEqual(code,2); self.assertIn('HIGH missing-frontmatter',out)
|
|
47
|
+
def test_missing_reference(self):
|
|
48
|
+
self.write(body='Synthetic instructions with enough detail. scripts/missing.sh')
|
|
49
|
+
self.assertEqual(self.run_cli('audit',str(self.root))[0],2)
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
import tempfile
|
|
2
|
+
import unittest
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from unittest.mock import patch
|
|
5
|
+
from skill_drift.skills import discover, read_skill
|
|
6
|
+
|
|
7
|
+
class DiscoveryTests(unittest.TestCase):
|
|
8
|
+
def setUp(self):
|
|
9
|
+
self.tmp = tempfile.TemporaryDirectory()
|
|
10
|
+
self.addCleanup(self.tmp.cleanup)
|
|
11
|
+
self.root = Path(self.tmp.name)
|
|
12
|
+
def skill(self, path='demo', text=None):
|
|
13
|
+
p = self.root / path
|
|
14
|
+
p.mkdir(parents=True, exist_ok=True)
|
|
15
|
+
(p/'SKILL.md').write_text(text if text is not None else '---\nname: demo\ndescription: test\n---\nBody', encoding='utf-8')
|
|
16
|
+
return p
|
|
17
|
+
def test_root(self):
|
|
18
|
+
self.skill('.')
|
|
19
|
+
self.assertEqual(discover(self.root), [self.root])
|
|
20
|
+
def test_nested(self):
|
|
21
|
+
self.assertEqual(discover(self.root), [])
|
|
22
|
+
self.skill('a/b')
|
|
23
|
+
self.assertEqual(len(discover(self.root)), 1)
|
|
24
|
+
def test_hidden(self):
|
|
25
|
+
self.skill('.claude/skills/demo')
|
|
26
|
+
self.skill('.cursor/skills/demo')
|
|
27
|
+
self.assertEqual(len(discover(self.root)), 2)
|
|
28
|
+
def test_skips(self):
|
|
29
|
+
for d in ('.git','node_modules','dist','__pycache__','.venv','venv'):
|
|
30
|
+
self.skill(d+'/demo')
|
|
31
|
+
self.assertEqual(discover(self.root), [])
|
|
32
|
+
def test_exact_case(self):
|
|
33
|
+
(self.root/'skill.md').write_text('test')
|
|
34
|
+
self.assertEqual(discover(self.root), [])
|
|
35
|
+
def test_sorted(self):
|
|
36
|
+
self.skill('z'); self.skill('a')
|
|
37
|
+
self.assertEqual([p.name for p in discover(self.root)], ['a','z'])
|
|
38
|
+
def test_bad_path(self):
|
|
39
|
+
with self.assertRaises(ValueError): discover(self.root/'absent')
|
|
40
|
+
def test_bad_utf8(self):
|
|
41
|
+
p = self.skill(); (p/'SKILL.md').write_bytes(b'\xff')
|
|
42
|
+
self.assertTrue(read_skill(p).error)
|
|
43
|
+
def test_permission_error(self):
|
|
44
|
+
p = self.skill()
|
|
45
|
+
with patch.object(Path, 'read_text', side_effect=PermissionError('denied')):
|
|
46
|
+
self.assertIn('denied', read_skill(p).error)
|
|
47
|
+
def test_quotes_lists(self):
|
|
48
|
+
s = read_skill(self.skill(text='---\nname: "demo"\ndescription: \'test\'\nfiles:\n - a.md\n - "b.md"\n---\nBody'))
|
|
49
|
+
self.assertFalse(s.error)
|
|
50
|
+
self.assertEqual(s.metadata['files'], ['a.md','b.md'])
|
|
51
|
+
def test_missing_close(self):
|
|
52
|
+
self.assertTrue(read_skill(self.skill(text='---\nname: demo')).error)
|
|
53
|
+
def test_complex_yaml(self):
|
|
54
|
+
for field in ('description: |','description: >','description: &anchor value','description: {a: b}','description: [a, b]','extra:\n nested: value','description: *anchor'):
|
|
55
|
+
with self.subTest(field=field):
|
|
56
|
+
self.assertTrue(read_skill(self.skill(text='---\nname: demo\n'+field+'\n---\nBody')).error)
|
|
57
|
+
def test_duplicate_key(self):
|
|
58
|
+
self.assertTrue(read_skill(self.skill(text='---\nname: a\nname: b\n---\nBody')).error)
|
|
59
|
+
def test_unterminated_quote(self):
|
|
60
|
+
self.assertTrue(read_skill(self.skill(text='---\nname: "demo\n---\nBody')).error)
|