instruction-drift 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- instruction_drift-0.1.0/LICENSE +21 -0
- instruction_drift-0.1.0/PKG-INFO +110 -0
- instruction_drift-0.1.0/README.md +100 -0
- instruction_drift-0.1.0/pyproject.toml +17 -0
- instruction_drift-0.1.0/setup.cfg +4 -0
- instruction_drift-0.1.0/src/instruction_drift/__init__.py +3 -0
- instruction_drift-0.1.0/src/instruction_drift/__main__.py +3 -0
- instruction_drift-0.1.0/src/instruction_drift/audit.py +118 -0
- instruction_drift-0.1.0/src/instruction_drift/cli.py +33 -0
- instruction_drift-0.1.0/src/instruction_drift/extract.py +52 -0
- instruction_drift-0.1.0/src/instruction_drift.egg-info/PKG-INFO +110 -0
- instruction_drift-0.1.0/src/instruction_drift.egg-info/SOURCES.txt +16 -0
- instruction_drift-0.1.0/src/instruction_drift.egg-info/dependency_links.txt +1 -0
- instruction_drift-0.1.0/src/instruction_drift.egg-info/entry_points.txt +2 -0
- instruction_drift-0.1.0/src/instruction_drift.egg-info/top_level.txt +1 -0
- instruction_drift-0.1.0/tests/test_audit.py +167 -0
- instruction_drift-0.1.0/tests/test_cli.py +87 -0
- instruction_drift-0.1.0/tests/test_extract.py +57 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 instruction-drift contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: instruction-drift
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Offline heuristic checks for stale repository instructions
|
|
5
|
+
License: MIT
|
|
6
|
+
Requires-Python: >=3.9
|
|
7
|
+
Description-Content-Type: text/markdown
|
|
8
|
+
License-File: LICENSE
|
|
9
|
+
Dynamic: license-file
|
|
10
|
+
|
|
11
|
+
# instruction-drift
|
|
12
|
+
|
|
13
|
+
An offline, deterministic, stdlib-only Python CLI that checks concrete instruction
|
|
14
|
+
claims against repository files. A finding means **check this claim**. It does
|
|
15
|
+
not establish that an instruction is wrong. No daemon, LLM, network, or keys.
|
|
16
|
+
|
|
17
|
+
Requires Python 3.9+. Install a locally built wheel with `pip install
|
|
18
|
+
/path/to/instruction_drift-0.1.0-py3-none-any.whl`.
|
|
19
|
+
|
|
20
|
+
```sh
|
|
21
|
+
instruction-drift audit .
|
|
22
|
+
instruction-drift audit examples/sample-repo --json
|
|
23
|
+
instruction-drift audit . --files AGENTS.md CONTRIBUTING.md --ignore 'docs/*'
|
|
24
|
+
instruction-drift audit . --fail-on medium
|
|
25
|
+
instruction-drift --version
|
|
26
|
+
instruction-drift --help
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
The repository defaults to the current directory. Default instruction files,
|
|
30
|
+
in order, are root AGENTS.md, CLAUDE.md, MUSE.md, and CONTRIBUTING.md. Missing
|
|
31
|
+
default files are skipped; an empty repository passes with a note. `--files`
|
|
32
|
+
replaces that list with paths relative to the repository. Missing explicitly
|
|
33
|
+
requested files, unreadable files, and invalid UTF-8 produce parse errors.
|
|
34
|
+
`--files` and `--ignore` accept multiple arguments and can be repeated.
|
|
35
|
+
|
|
36
|
+
Checks are heuristic:
|
|
37
|
+
|
|
38
|
+
- **referenced-file (HIGH):** missing path tokens containing `/` and ending in
|
|
39
|
+
.py, .js, .ts, .md, .json, .yaml, .yml, .toml, .sh, .go, .rs, or .txt;
|
|
40
|
+
also directory references rooted under src/, tests/, docs/, examples/,
|
|
41
|
+
scripts/, or bin/. Trailing `/` requires a directory. Paths resolve relative
|
|
42
|
+
to the repository root. URLs, tokens containing `*`, and paths resolving
|
|
43
|
+
outside the repository are skipped.
|
|
44
|
+
- **script-in-command (HIGH):** missing path tokens in backticked commands;
|
|
45
|
+
missing `make target` targets when a root Makefile exists. Simple literal
|
|
46
|
+
target definitions are recognized; Makefile variables and comments are not
|
|
47
|
+
definitions. A command path is reported once rather than also as a reference.
|
|
48
|
+
- **prohibition-contradiction (MEDIUM):** “do not use”, “never use”, “must not
|
|
49
|
+
use”, or “avoid using” followed by an identifier also found in a Python
|
|
50
|
+
import/from import or JS/TS require/from import. Verify manually: exceptions
|
|
51
|
+
and test fixtures may be intentional. Instruction files are excluded from
|
|
52
|
+
source search. Hidden directories, .git, node_modules, dist, __pycache__,
|
|
53
|
+
.venv, and venv are skipped, as are symlinked directories.
|
|
54
|
+
|
|
55
|
+
The report includes files read, file:line locations, kinds, claims, severities,
|
|
56
|
+
details, totals by kind and severity, ignored count, parse-error count, and a
|
|
57
|
+
PASS/WARN/FAIL verdict. `--json` exposes the same information as valid JSON.
|
|
58
|
+
Claims matching `--ignore` glob patterns are excluded and counted before
|
|
59
|
+
checking their validity. Findings are deduplicated by file, line, kind, and
|
|
60
|
+
claim. At most 500 findings are retained in instruction-file and line order;
|
|
61
|
+
additional findings set the truncation flag. Totals describe retained findings. Failure thresholds still apply to findings
|
|
62
|
+
beyond the display cap.
|
|
63
|
+
|
|
64
|
+
Exit codes:
|
|
65
|
+
|
|
66
|
+
| Code | Meaning |
|
|
67
|
+
| --- | --- |
|
|
68
|
+
| 0 | PASS: no findings and no parse errors |
|
|
69
|
+
| 1 | WARN: findings exist, but none reach the failure threshold |
|
|
70
|
+
| 2 | FAIL: a finding reaches the threshold, or any parse error occurs |
|
|
71
|
+
|
|
72
|
+
The default threshold is HIGH; `--fail-on medium` also fails MEDIUM findings.
|
|
73
|
+
For example, the synthetic sample repo cites a renamed script and moved test
|
|
74
|
+
directory, and prohibits an identifier imported in src/app.py. Auditing it
|
|
75
|
+
returns two HIGH findings, one MEDIUM finding, and exit code 2. Its illustrative
|
|
76
|
+
import does not require an installed dependency because commands are never run.
|
|
77
|
+
|
|
78
|
+
## Limitations
|
|
79
|
+
|
|
80
|
+
This is pattern matching, not semantic judgement. Paraphrased claims are not
|
|
81
|
+
checked. A MEDIUM contradiction needs human review. Commands are not executed;
|
|
82
|
+
only path tokens and simple Makefile target names are checked. Dynamic imports,
|
|
83
|
+
complex shell syntax, generated paths, multiline imports, Makefile includes and
|
|
84
|
+
computed targets, and language-specific resolution are not interpreted.
|
|
85
|
+
Lexical import patterns may match comments or strings and may miss real imports.
|
|
86
|
+
Source files that cannot be decoded or read are skipped; this can hide import
|
|
87
|
+
matches. Instruction-file read failures always produce parse errors. A truncated
|
|
88
|
+
report is incomplete. Filesystem changes during a scan can affect its results.
|
|
89
|
+
|
|
90
|
+
## Differentiation
|
|
91
|
+
|
|
92
|
+
- **startup-trust-guard** audits TRUST: can an attacker plant or rewrite the
|
|
93
|
+
instruction file? instruction-drift audits TRUTH: is what it says still true
|
|
94
|
+
about the on-disk repository?
|
|
95
|
+
- **rules-doctor** audits rule-file health and structure. This tool cross-checks
|
|
96
|
+
instruction claims against the on-disk repo.
|
|
97
|
+
- **edit-guard** blocks edits made on stale reads. This tool finds stale
|
|
98
|
+
instructions that can cause those edits.
|
|
99
|
+
- **context-mode / heavy runtimes:** this is an offline stdlib CLI with no
|
|
100
|
+
daemon, no LLM, and no network.
|
|
101
|
+
|
|
102
|
+
## Development
|
|
103
|
+
|
|
104
|
+
```sh
|
|
105
|
+
PYTHONPATH=src python3 -m unittest discover -s tests -v
|
|
106
|
+
python3 -m build
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
Building requires setuptools, wheel, and the build frontend already installed
|
|
110
|
+
for offline use. Runtime and tests use only the Python standard library.
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
# instruction-drift
|
|
2
|
+
|
|
3
|
+
An offline, deterministic, stdlib-only Python CLI that checks concrete instruction
|
|
4
|
+
claims against repository files. A finding means **check this claim**. It does
|
|
5
|
+
not establish that an instruction is wrong. No daemon, LLM, network, or keys.
|
|
6
|
+
|
|
7
|
+
Requires Python 3.9+. Install a locally built wheel with `pip install
|
|
8
|
+
/path/to/instruction_drift-0.1.0-py3-none-any.whl`.
|
|
9
|
+
|
|
10
|
+
```sh
|
|
11
|
+
instruction-drift audit .
|
|
12
|
+
instruction-drift audit examples/sample-repo --json
|
|
13
|
+
instruction-drift audit . --files AGENTS.md CONTRIBUTING.md --ignore 'docs/*'
|
|
14
|
+
instruction-drift audit . --fail-on medium
|
|
15
|
+
instruction-drift --version
|
|
16
|
+
instruction-drift --help
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
The repository defaults to the current directory. Default instruction files,
|
|
20
|
+
in order, are root AGENTS.md, CLAUDE.md, MUSE.md, and CONTRIBUTING.md. Missing
|
|
21
|
+
default files are skipped; an empty repository passes with a note. `--files`
|
|
22
|
+
replaces that list with paths relative to the repository. Missing explicitly
|
|
23
|
+
requested files, unreadable files, and invalid UTF-8 produce parse errors.
|
|
24
|
+
`--files` and `--ignore` accept multiple arguments and can be repeated.
|
|
25
|
+
|
|
26
|
+
Checks are heuristic:
|
|
27
|
+
|
|
28
|
+
- **referenced-file (HIGH):** missing path tokens containing `/` and ending in
|
|
29
|
+
.py, .js, .ts, .md, .json, .yaml, .yml, .toml, .sh, .go, .rs, or .txt;
|
|
30
|
+
also directory references rooted under src/, tests/, docs/, examples/,
|
|
31
|
+
scripts/, or bin/. Trailing `/` requires a directory. Paths resolve relative
|
|
32
|
+
to the repository root. URLs, tokens containing `*`, and paths resolving
|
|
33
|
+
outside the repository are skipped.
|
|
34
|
+
- **script-in-command (HIGH):** missing path tokens in backticked commands;
|
|
35
|
+
missing `make target` targets when a root Makefile exists. Simple literal
|
|
36
|
+
target definitions are recognized; Makefile variables and comments are not
|
|
37
|
+
definitions. A command path is reported once rather than also as a reference.
|
|
38
|
+
- **prohibition-contradiction (MEDIUM):** “do not use”, “never use”, “must not
|
|
39
|
+
use”, or “avoid using” followed by an identifier also found in a Python
|
|
40
|
+
import/from import or JS/TS require/from import. Verify manually: exceptions
|
|
41
|
+
and test fixtures may be intentional. Instruction files are excluded from
|
|
42
|
+
source search. Hidden directories, .git, node_modules, dist, __pycache__,
|
|
43
|
+
.venv, and venv are skipped, as are symlinked directories.
|
|
44
|
+
|
|
45
|
+
The report includes files read, file:line locations, kinds, claims, severities,
|
|
46
|
+
details, totals by kind and severity, ignored count, parse-error count, and a
|
|
47
|
+
PASS/WARN/FAIL verdict. `--json` exposes the same information as valid JSON.
|
|
48
|
+
Claims matching `--ignore` glob patterns are excluded and counted before
|
|
49
|
+
checking their validity. Findings are deduplicated by file, line, kind, and
|
|
50
|
+
claim. At most 500 findings are retained in instruction-file and line order;
|
|
51
|
+
additional findings set the truncation flag. Totals describe retained findings. Failure thresholds still apply to findings
|
|
52
|
+
beyond the display cap.
|
|
53
|
+
|
|
54
|
+
Exit codes:
|
|
55
|
+
|
|
56
|
+
| Code | Meaning |
|
|
57
|
+
| --- | --- |
|
|
58
|
+
| 0 | PASS: no findings and no parse errors |
|
|
59
|
+
| 1 | WARN: findings exist, but none reach the failure threshold |
|
|
60
|
+
| 2 | FAIL: a finding reaches the threshold, or any parse error occurs |
|
|
61
|
+
|
|
62
|
+
The default threshold is HIGH; `--fail-on medium` also fails MEDIUM findings.
|
|
63
|
+
For example, the synthetic sample repo cites a renamed script and moved test
|
|
64
|
+
directory, and prohibits an identifier imported in src/app.py. Auditing it
|
|
65
|
+
returns two HIGH findings, one MEDIUM finding, and exit code 2. Its illustrative
|
|
66
|
+
import does not require an installed dependency because commands are never run.
|
|
67
|
+
|
|
68
|
+
## Limitations
|
|
69
|
+
|
|
70
|
+
This is pattern matching, not semantic judgement. Paraphrased claims are not
|
|
71
|
+
checked. A MEDIUM contradiction needs human review. Commands are not executed;
|
|
72
|
+
only path tokens and simple Makefile target names are checked. Dynamic imports,
|
|
73
|
+
complex shell syntax, generated paths, multiline imports, Makefile includes and
|
|
74
|
+
computed targets, and language-specific resolution are not interpreted.
|
|
75
|
+
Lexical import patterns may match comments or strings and may miss real imports.
|
|
76
|
+
Source files that cannot be decoded or read are skipped; this can hide import
|
|
77
|
+
matches. Instruction-file read failures always produce parse errors. A truncated
|
|
78
|
+
report is incomplete. Filesystem changes during a scan can affect its results.
|
|
79
|
+
|
|
80
|
+
## Differentiation
|
|
81
|
+
|
|
82
|
+
- **startup-trust-guard** audits TRUST: can an attacker plant or rewrite the
|
|
83
|
+
instruction file? instruction-drift audits TRUTH: is what it says still true
|
|
84
|
+
about the on-disk repository?
|
|
85
|
+
- **rules-doctor** audits rule-file health and structure. This tool cross-checks
|
|
86
|
+
instruction claims against the on-disk repo.
|
|
87
|
+
- **edit-guard** blocks edits made on stale reads. This tool finds stale
|
|
88
|
+
instructions that can cause those edits.
|
|
89
|
+
- **context-mode / heavy runtimes:** this is an offline stdlib CLI with no
|
|
90
|
+
daemon, no LLM, and no network.
|
|
91
|
+
|
|
92
|
+
## Development
|
|
93
|
+
|
|
94
|
+
```sh
|
|
95
|
+
PYTHONPATH=src python3 -m unittest discover -s tests -v
|
|
96
|
+
python3 -m build
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
Building requires setuptools, wheel, and the build frontend already installed
|
|
100
|
+
for offline use. Runtime and tests use only the Python standard library.
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=61", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "instruction-drift"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Offline heuristic checks for stale repository instructions"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.9"
|
|
11
|
+
license = {text = "MIT"}
|
|
12
|
+
|
|
13
|
+
[project.scripts]
|
|
14
|
+
instruction-drift = "instruction_drift.cli:main"
|
|
15
|
+
|
|
16
|
+
[tool.setuptools.packages.find]
|
|
17
|
+
where = ["src"]
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
"""Deterministic, offline comparison of lexical claims with disk contents."""
|
|
2
|
+
from dataclasses import asdict, dataclass
|
|
3
|
+
from fnmatch import fnmatchcase
|
|
4
|
+
import os
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
import re
|
|
7
|
+
from .extract import extract
|
|
8
|
+
|
|
9
|
+
DEFAULT_FILES = ('AGENTS.md', 'CLAUDE.md', 'MUSE.md', 'CONTRIBUTING.md')
|
|
10
|
+
SKIP = {'node_modules', 'dist', '__pycache__', 'venv'}
|
|
11
|
+
|
|
12
|
+
@dataclass(frozen=True)
|
|
13
|
+
class Finding:
|
|
14
|
+
file: str
|
|
15
|
+
line: int
|
|
16
|
+
kind: str
|
|
17
|
+
claim: str
|
|
18
|
+
severity: str
|
|
19
|
+
detail: str
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def imports(root, excluded):
|
|
23
|
+
identifiers = set()
|
|
24
|
+
for directory, dirs, files in os.walk(root):
|
|
25
|
+
dirs[:] = sorted(d for d in dirs if not d.startswith('.') and d not in SKIP
|
|
26
|
+
and not (Path(directory) / d).is_symlink())
|
|
27
|
+
for name in sorted(files):
|
|
28
|
+
path = Path(directory) / name
|
|
29
|
+
if path.resolve() in excluded or path.suffix not in ('.py', '.js', '.ts'):
|
|
30
|
+
continue
|
|
31
|
+
try:
|
|
32
|
+
content = path.read_text(encoding='utf-8')
|
|
33
|
+
except (OSError, UnicodeError):
|
|
34
|
+
continue
|
|
35
|
+
if path.suffix == '.py':
|
|
36
|
+
for match in re.finditer(r'^\s*(?:from\s+([\w.]+)\s+import\b|import\s+([^\n;#]+))', content, re.M):
|
|
37
|
+
if match.group(1):
|
|
38
|
+
identifiers.add(match.group(1))
|
|
39
|
+
else:
|
|
40
|
+
identifiers.update(part.strip().split()[0] for part in
|
|
41
|
+
match.group(2).split(',') if part.strip())
|
|
42
|
+
else:
|
|
43
|
+
identifiers.update(re.findall(r'''(?:require\(\s*|\bfrom\s*)['"]([^'"]+)['"]''', content))
|
|
44
|
+
return identifiers
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def audit(repo='.', files=None, ignore=(), fail_on='high'):
|
|
48
|
+
root = Path(repo).resolve()
|
|
49
|
+
names = list(DEFAULT_FILES if files is None else files)
|
|
50
|
+
report = dict(files_read=[], findings=[], ignored=0, parse_errors=[], truncated=False)
|
|
51
|
+
if not root.is_dir():
|
|
52
|
+
report['parse_errors'].append({'file': str(repo), 'detail': 'Repository is not a directory'})
|
|
53
|
+
excluded = {(root / name).resolve() for name in names}
|
|
54
|
+
used = imports(root, excluded) if root.is_dir() else set()
|
|
55
|
+
targets = None
|
|
56
|
+
seen = set()
|
|
57
|
+
threshold_hit = False
|
|
58
|
+
for name in names if root.is_dir() else []:
|
|
59
|
+
path = root / name
|
|
60
|
+
if files is None and not path.exists():
|
|
61
|
+
continue
|
|
62
|
+
try:
|
|
63
|
+
content = path.read_text(encoding='utf-8')
|
|
64
|
+
except (OSError, UnicodeError) as error:
|
|
65
|
+
report['parse_errors'].append({'file': name, 'detail': str(error)})
|
|
66
|
+
continue
|
|
67
|
+
report['files_read'].append(name)
|
|
68
|
+
for item in extract(content):
|
|
69
|
+
key = (name, item.line, item.kind, item.claim)
|
|
70
|
+
if key in seen:
|
|
71
|
+
continue
|
|
72
|
+
seen.add(key)
|
|
73
|
+
if any(fnmatchcase(item.claim, pattern) for pattern in ignore):
|
|
74
|
+
report['ignored'] += 1
|
|
75
|
+
continue
|
|
76
|
+
severity, detail = 'HIGH', ''
|
|
77
|
+
if item.kind == 'prohibition-contradiction':
|
|
78
|
+
if item.claim not in used:
|
|
79
|
+
continue
|
|
80
|
+
severity = 'MEDIUM'
|
|
81
|
+
detail = 'Identifier appears in source imports; verify manually.'
|
|
82
|
+
elif item.claim.startswith('make '):
|
|
83
|
+
makefile = root / 'Makefile'
|
|
84
|
+
if not makefile.exists():
|
|
85
|
+
continue
|
|
86
|
+
if targets is None:
|
|
87
|
+
try:
|
|
88
|
+
targets = set(re.findall(r'^([A-Za-z0-9_.-]+)\s*:(?!=)', makefile.read_text(encoding='utf-8'), re.M))
|
|
89
|
+
except (OSError, UnicodeError) as error:
|
|
90
|
+
report['parse_errors'].append({'file': 'Makefile', 'detail': str(error)})
|
|
91
|
+
targets = set()
|
|
92
|
+
if item.claim[5:] in targets:
|
|
93
|
+
continue
|
|
94
|
+
detail = 'Target not found in root Makefile.'
|
|
95
|
+
else:
|
|
96
|
+
candidate = root / item.claim
|
|
97
|
+
try:
|
|
98
|
+
candidate.resolve().relative_to(root)
|
|
99
|
+
except ValueError:
|
|
100
|
+
continue
|
|
101
|
+
exists = candidate.is_dir() if item.claim.endswith('/') else candidate.exists()
|
|
102
|
+
if exists:
|
|
103
|
+
continue
|
|
104
|
+
detail = 'Referenced path does not exist' + (' as a directory.' if item.claim.endswith('/') else '.')
|
|
105
|
+
threshold_hit = threshold_hit or severity == 'HIGH' or fail_on == 'medium'
|
|
106
|
+
if len(report['findings']) < 500:
|
|
107
|
+
report['findings'].append(asdict(Finding(name, item.line, item.kind, item.claim, severity, detail)))
|
|
108
|
+
else:
|
|
109
|
+
report['truncated'] = True
|
|
110
|
+
findings = report['findings']
|
|
111
|
+
report['totals'] = {'kind': {k: sum(f['kind'] == k for f in findings) for k in
|
|
112
|
+
('referenced-file', 'script-in-command', 'prohibition-contradiction')},
|
|
113
|
+
'severity': {s: sum(f['severity'] == s for f in findings) for s in ('HIGH', 'MEDIUM')}}
|
|
114
|
+
failed = report['parse_errors'] or threshold_hit
|
|
115
|
+
report['exit_code'] = 2 if failed else (1 if findings else 0)
|
|
116
|
+
report['verdict'] = {0: 'PASS', 1: 'WARN', 2: 'FAIL'}[report['exit_code']]
|
|
117
|
+
report['parse_error_count'] = len(report['parse_errors'])
|
|
118
|
+
return report
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
"""Command-line interface; commands mentioned in instructions are never run."""
|
|
2
|
+
import argparse
|
|
3
|
+
import json
|
|
4
|
+
from . import __version__
|
|
5
|
+
from .audit import audit
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def main(argv=None):
|
|
9
|
+
parser = argparse.ArgumentParser(prog='instruction-drift', description='Check repository instruction claims offline.')
|
|
10
|
+
parser.add_argument('--version', action='version', version='%(prog)s ' + __version__)
|
|
11
|
+
sub = parser.add_subparsers(dest='command', required=True)
|
|
12
|
+
check = sub.add_parser('audit', help='Audit instruction claims against repository files')
|
|
13
|
+
check.add_argument('repo', nargs='?', default='.')
|
|
14
|
+
check.add_argument('--files', nargs='+', action='extend')
|
|
15
|
+
check.add_argument('--ignore', nargs='+', action='extend', default=[])
|
|
16
|
+
check.add_argument('--json', action='store_true')
|
|
17
|
+
check.add_argument('--fail-on', choices=('high', 'medium'), default='high')
|
|
18
|
+
args = parser.parse_args(argv)
|
|
19
|
+
report = audit(args.repo, args.files, args.ignore, args.fail_on)
|
|
20
|
+
if args.json:
|
|
21
|
+
print(json.dumps(report, indent=2))
|
|
22
|
+
else:
|
|
23
|
+
print('Instruction files read: ' + (', '.join(report['files_read']) or 'none (no instruction files read)'))
|
|
24
|
+
for finding in report['findings']:
|
|
25
|
+
print('{file}:{line}: {severity} {kind}: {claim} — {detail}'.format(**finding))
|
|
26
|
+
for error in report['parse_errors']:
|
|
27
|
+
print('{file}: parse error: {detail}'.format(**error))
|
|
28
|
+
print('Totals: ' + json.dumps(report['totals'], sort_keys=True))
|
|
29
|
+
print('Ignored: {}; parse errors: {}'.format(report['ignored'], report['parse_error_count']))
|
|
30
|
+
if report['truncated']:
|
|
31
|
+
print('Report truncated at 500 findings.')
|
|
32
|
+
print(report['verdict'])
|
|
33
|
+
return report['exit_code']
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
"""Small, deliberately lexical instruction-claim extractors."""
|
|
2
|
+
from dataclasses import dataclass
|
|
3
|
+
import re
|
|
4
|
+
|
|
5
|
+
EXTENSIONS = ('.py', '.js', '.ts', '.md', '.json', '.yaml', '.yml',
|
|
6
|
+
'.toml', '.sh', '.go', '.rs', '.txt')
|
|
7
|
+
ROOTS = ('src/', 'tests/', 'docs/', 'examples/', 'scripts/', 'bin/')
|
|
8
|
+
TOKEN = re.compile(r"[^\s`\"'<>|;()]+")
|
|
9
|
+
BACKTICK = re.compile(r'`([^`]+)`')
|
|
10
|
+
PROHIBITION = re.compile(
|
|
11
|
+
r'\b(?:do not use|never use|must not use|avoid using)\s+`?([\w@][\w.\-@]*)',
|
|
12
|
+
re.IGNORECASE)
|
|
13
|
+
|
|
14
|
+
@dataclass(frozen=True)
|
|
15
|
+
class Claim:
|
|
16
|
+
line: int
|
|
17
|
+
kind: str
|
|
18
|
+
claim: str
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def path_token(token):
|
|
22
|
+
"""Return a normalized path claim, or None for unsupported tokens."""
|
|
23
|
+
token = token.lstrip(',:!?[]{}').rstrip('.,:!?[]{}')
|
|
24
|
+
if '*' in token or '://' in token:
|
|
25
|
+
return None
|
|
26
|
+
bare = token[2:] if token.startswith('./') else token
|
|
27
|
+
if '/' in bare and (bare.endswith(EXTENSIONS) or bare.startswith(ROOTS)):
|
|
28
|
+
return token
|
|
29
|
+
return None
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def extract(text):
|
|
33
|
+
"""Extract claims in line order; command paths have a single kind."""
|
|
34
|
+
result = []
|
|
35
|
+
for number, line in enumerate(text.splitlines(), 1):
|
|
36
|
+
spans = list(BACKTICK.finditer(line))
|
|
37
|
+
for match in TOKEN.finditer(line):
|
|
38
|
+
path = path_token(match.group())
|
|
39
|
+
if path:
|
|
40
|
+
command = any(s.start(1) <= match.start() < s.end(1)
|
|
41
|
+
and (len(s.group(1).split()) > 1 or s.group(1).startswith('./'))
|
|
42
|
+
for s in spans)
|
|
43
|
+
result.append(Claim(number, 'script-in-command' if command else
|
|
44
|
+
'referenced-file', path))
|
|
45
|
+
for span in spans:
|
|
46
|
+
words = span.group(1).split()
|
|
47
|
+
if len(words) >= 2 and words[0] == 'make' and not words[1].startswith('-'):
|
|
48
|
+
result.append(Claim(number, 'script-in-command', 'make ' + words[1]))
|
|
49
|
+
for match in PROHIBITION.finditer(line):
|
|
50
|
+
result.append(Claim(number, 'prohibition-contradiction',
|
|
51
|
+
match.group(1).rstrip('.')))
|
|
52
|
+
return list(dict.fromkeys(result))
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: instruction-drift
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Offline heuristic checks for stale repository instructions
|
|
5
|
+
License: MIT
|
|
6
|
+
Requires-Python: >=3.9
|
|
7
|
+
Description-Content-Type: text/markdown
|
|
8
|
+
License-File: LICENSE
|
|
9
|
+
Dynamic: license-file
|
|
10
|
+
|
|
11
|
+
# instruction-drift
|
|
12
|
+
|
|
13
|
+
An offline, deterministic, stdlib-only Python CLI that checks concrete instruction
|
|
14
|
+
claims against repository files. A finding means **check this claim**. It does
|
|
15
|
+
not establish that an instruction is wrong. No daemon, LLM, network, or keys.
|
|
16
|
+
|
|
17
|
+
Requires Python 3.9+. Install a locally built wheel with `pip install
|
|
18
|
+
/path/to/instruction_drift-0.1.0-py3-none-any.whl`.
|
|
19
|
+
|
|
20
|
+
```sh
|
|
21
|
+
instruction-drift audit .
|
|
22
|
+
instruction-drift audit examples/sample-repo --json
|
|
23
|
+
instruction-drift audit . --files AGENTS.md CONTRIBUTING.md --ignore 'docs/*'
|
|
24
|
+
instruction-drift audit . --fail-on medium
|
|
25
|
+
instruction-drift --version
|
|
26
|
+
instruction-drift --help
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
The repository defaults to the current directory. Default instruction files,
|
|
30
|
+
in order, are root AGENTS.md, CLAUDE.md, MUSE.md, and CONTRIBUTING.md. Missing
|
|
31
|
+
default files are skipped; an empty repository passes with a note. `--files`
|
|
32
|
+
replaces that list with paths relative to the repository. Missing explicitly
|
|
33
|
+
requested files, unreadable files, and invalid UTF-8 produce parse errors.
|
|
34
|
+
`--files` and `--ignore` accept multiple arguments and can be repeated.
|
|
35
|
+
|
|
36
|
+
Checks are heuristic:
|
|
37
|
+
|
|
38
|
+
- **referenced-file (HIGH):** missing path tokens containing `/` and ending in
|
|
39
|
+
.py, .js, .ts, .md, .json, .yaml, .yml, .toml, .sh, .go, .rs, or .txt;
|
|
40
|
+
also directory references rooted under src/, tests/, docs/, examples/,
|
|
41
|
+
scripts/, or bin/. Trailing `/` requires a directory. Paths resolve relative
|
|
42
|
+
to the repository root. URLs, tokens containing `*`, and paths resolving
|
|
43
|
+
outside the repository are skipped.
|
|
44
|
+
- **script-in-command (HIGH):** missing path tokens in backticked commands;
|
|
45
|
+
missing `make target` targets when a root Makefile exists. Simple literal
|
|
46
|
+
target definitions are recognized; Makefile variables and comments are not
|
|
47
|
+
definitions. A command path is reported once rather than also as a reference.
|
|
48
|
+
- **prohibition-contradiction (MEDIUM):** “do not use”, “never use”, “must not
|
|
49
|
+
use”, or “avoid using” followed by an identifier also found in a Python
|
|
50
|
+
import/from import or JS/TS require/from import. Verify manually: exceptions
|
|
51
|
+
and test fixtures may be intentional. Instruction files are excluded from
|
|
52
|
+
source search. Hidden directories, .git, node_modules, dist, __pycache__,
|
|
53
|
+
.venv, and venv are skipped, as are symlinked directories.
|
|
54
|
+
|
|
55
|
+
The report includes files read, file:line locations, kinds, claims, severities,
|
|
56
|
+
details, totals by kind and severity, ignored count, parse-error count, and a
|
|
57
|
+
PASS/WARN/FAIL verdict. `--json` exposes the same information as valid JSON.
|
|
58
|
+
Claims matching `--ignore` glob patterns are excluded and counted before
|
|
59
|
+
checking their validity. Findings are deduplicated by file, line, kind, and
|
|
60
|
+
claim. At most 500 findings are retained in instruction-file and line order;
|
|
61
|
+
additional findings set the truncation flag. Totals describe retained findings. Failure thresholds still apply to findings
|
|
62
|
+
beyond the display cap.
|
|
63
|
+
|
|
64
|
+
Exit codes:
|
|
65
|
+
|
|
66
|
+
| Code | Meaning |
|
|
67
|
+
| --- | --- |
|
|
68
|
+
| 0 | PASS: no findings and no parse errors |
|
|
69
|
+
| 1 | WARN: findings exist, but none reach the failure threshold |
|
|
70
|
+
| 2 | FAIL: a finding reaches the threshold, or any parse error occurs |
|
|
71
|
+
|
|
72
|
+
The default threshold is HIGH; `--fail-on medium` also fails MEDIUM findings.
|
|
73
|
+
For example, the synthetic sample repo cites a renamed script and moved test
|
|
74
|
+
directory, and prohibits an identifier imported in src/app.py. Auditing it
|
|
75
|
+
returns two HIGH findings, one MEDIUM finding, and exit code 2. Its illustrative
|
|
76
|
+
import does not require an installed dependency because commands are never run.
|
|
77
|
+
|
|
78
|
+
## Limitations
|
|
79
|
+
|
|
80
|
+
This is pattern matching, not semantic judgement. Paraphrased claims are not
|
|
81
|
+
checked. A MEDIUM contradiction needs human review. Commands are not executed;
|
|
82
|
+
only path tokens and simple Makefile target names are checked. Dynamic imports,
|
|
83
|
+
complex shell syntax, generated paths, multiline imports, Makefile includes and
|
|
84
|
+
computed targets, and language-specific resolution are not interpreted.
|
|
85
|
+
Lexical import patterns may match comments or strings and may miss real imports.
|
|
86
|
+
Source files that cannot be decoded or read are skipped; this can hide import
|
|
87
|
+
matches. Instruction-file read failures always produce parse errors. A truncated
|
|
88
|
+
report is incomplete. Filesystem changes during a scan can affect its results.
|
|
89
|
+
|
|
90
|
+
## Differentiation
|
|
91
|
+
|
|
92
|
+
- **startup-trust-guard** audits TRUST: can an attacker plant or rewrite the
|
|
93
|
+
instruction file? instruction-drift audits TRUTH: is what it says still true
|
|
94
|
+
about the on-disk repository?
|
|
95
|
+
- **rules-doctor** audits rule-file health and structure. This tool cross-checks
|
|
96
|
+
instruction claims against the on-disk repo.
|
|
97
|
+
- **edit-guard** blocks edits made on stale reads. This tool finds stale
|
|
98
|
+
instructions that can cause those edits.
|
|
99
|
+
- **context-mode / heavy runtimes:** this is an offline stdlib CLI with no
|
|
100
|
+
daemon, no LLM, and no network.
|
|
101
|
+
|
|
102
|
+
## Development
|
|
103
|
+
|
|
104
|
+
```sh
|
|
105
|
+
PYTHONPATH=src python3 -m unittest discover -s tests -v
|
|
106
|
+
python3 -m build
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
Building requires setuptools, wheel, and the build frontend already installed
|
|
110
|
+
for offline use. Runtime and tests use only the Python standard library.
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
README.md
|
|
3
|
+
pyproject.toml
|
|
4
|
+
src/instruction_drift/__init__.py
|
|
5
|
+
src/instruction_drift/__main__.py
|
|
6
|
+
src/instruction_drift/audit.py
|
|
7
|
+
src/instruction_drift/cli.py
|
|
8
|
+
src/instruction_drift/extract.py
|
|
9
|
+
src/instruction_drift.egg-info/PKG-INFO
|
|
10
|
+
src/instruction_drift.egg-info/SOURCES.txt
|
|
11
|
+
src/instruction_drift.egg-info/dependency_links.txt
|
|
12
|
+
src/instruction_drift.egg-info/entry_points.txt
|
|
13
|
+
src/instruction_drift.egg-info/top_level.txt
|
|
14
|
+
tests/test_audit.py
|
|
15
|
+
tests/test_cli.py
|
|
16
|
+
tests/test_extract.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
instruction_drift
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
from pathlib import Path
|
|
2
|
+
import tempfile
|
|
3
|
+
import unittest
|
|
4
|
+
from unittest.mock import patch
|
|
5
|
+
from instruction_drift.audit import audit
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class AuditTests(unittest.TestCase):
|
|
9
|
+
def setUp(self):
|
|
10
|
+
self.temp = tempfile.TemporaryDirectory()
|
|
11
|
+
self.addCleanup(self.temp.cleanup)
|
|
12
|
+
self.root = Path(self.temp.name)
|
|
13
|
+
|
|
14
|
+
def write(self, name, content=''):
|
|
15
|
+
path = self.root / name
|
|
16
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
17
|
+
path.write_text(content, encoding='utf-8')
|
|
18
|
+
return path
|
|
19
|
+
|
|
20
|
+
def check(self, content, **kwargs):
|
|
21
|
+
self.write('AGENTS.md', content)
|
|
22
|
+
return audit(self.root, **kwargs)
|
|
23
|
+
|
|
24
|
+
def test_empty(self):
|
|
25
|
+
self.assertEqual(audit(self.root)['exit_code'], 0)
|
|
26
|
+
|
|
27
|
+
def test_exists(self):
|
|
28
|
+
self.write('src/app.py')
|
|
29
|
+
self.assertEqual(self.check('src/app.py')['findings'], [])
|
|
30
|
+
|
|
31
|
+
def test_missing(self):
|
|
32
|
+
r = self.check('src/missing.py')
|
|
33
|
+
self.assertEqual((r['exit_code'], r['totals']['severity']['HIGH']), (2, 1))
|
|
34
|
+
|
|
35
|
+
def test_directory_exists(self):
|
|
36
|
+
(self.root / 'tests/unit').mkdir(parents=True)
|
|
37
|
+
self.assertEqual(self.check('tests/unit/')['exit_code'], 0)
|
|
38
|
+
|
|
39
|
+
def test_directory_missing(self):
|
|
40
|
+
self.assertEqual(self.check('tests/old/')['exit_code'], 2)
|
|
41
|
+
|
|
42
|
+
def test_directory_is_file(self):
|
|
43
|
+
self.write('tests/unit')
|
|
44
|
+
self.assertEqual(self.check('tests/unit/')['exit_code'], 2)
|
|
45
|
+
|
|
46
|
+
def test_url_glob(self):
|
|
47
|
+
self.assertEqual(self.check('https://site/a.py src/*.py')['findings'], [])
|
|
48
|
+
|
|
49
|
+
def test_absolute_outside(self):
|
|
50
|
+
self.assertEqual(self.check('/outside/missing.py')['findings'], [])
|
|
51
|
+
|
|
52
|
+
def test_absolute_inside(self):
|
|
53
|
+
self.assertEqual(self.check(str(self.root / 'missing.py'))['exit_code'], 2)
|
|
54
|
+
|
|
55
|
+
def test_parent_outside(self):
|
|
56
|
+
self.assertEqual(self.check('../outside/a.py')['exit_code'], 0)
|
|
57
|
+
|
|
58
|
+
def test_command_missing(self):
|
|
59
|
+
r = self.check('`python scripts/run.py`')
|
|
60
|
+
self.assertEqual(r['totals']['kind']['script-in-command'], 1)
|
|
61
|
+
self.assertEqual(len(r['findings']), 1)
|
|
62
|
+
|
|
63
|
+
def test_command_exists(self):
|
|
64
|
+
self.write('scripts/run.sh')
|
|
65
|
+
self.assertEqual(self.check('`./scripts/run.sh`')['exit_code'], 0)
|
|
66
|
+
|
|
67
|
+
def test_make_present(self):
|
|
68
|
+
self.write('Makefile', '# fake:\nNAME := x\ncheck:\n\ttrue\n')
|
|
69
|
+
self.assertEqual(self.check('`make check`')['exit_code'], 0)
|
|
70
|
+
|
|
71
|
+
def test_make_absent(self):
|
|
72
|
+
self.write('Makefile', '# fake:\nNAME := x\ncheck:\n')
|
|
73
|
+
self.assertEqual(self.check('`make fake` `make NAME`')['totals']['severity']['HIGH'], 2)
|
|
74
|
+
|
|
75
|
+
def test_make_without_makefile(self):
|
|
76
|
+
self.assertEqual(self.check('`make check`')['exit_code'], 0)
|
|
77
|
+
|
|
78
|
+
def test_python_import(self):
|
|
79
|
+
self.write('src/a.py', 'import legacy\n')
|
|
80
|
+
self.assertEqual(self.check('Never use legacy')['exit_code'], 1)
|
|
81
|
+
|
|
82
|
+
def test_python_from(self):
|
|
83
|
+
self.write('src/a.py', 'from legacy import client\n')
|
|
84
|
+
self.assertEqual(self.check('Never use legacy')['exit_code'], 1)
|
|
85
|
+
|
|
86
|
+
def test_multiple_python_imports(self):
|
|
87
|
+
self.write('src/a.py', 'import os, legacy as old\n')
|
|
88
|
+
self.assertEqual(self.check('Never use legacy')['exit_code'], 1)
|
|
89
|
+
|
|
90
|
+
def test_js_ts(self):
|
|
91
|
+
for suffix, code in [('js', "const x = require('legacy');"), ('ts', "import x from 'legacy';")]:
|
|
92
|
+
with self.subTest(suffix=suffix):
|
|
93
|
+
self.write('src/a.' + suffix, code)
|
|
94
|
+
self.assertEqual(self.check('Never use legacy')['exit_code'], 1)
|
|
95
|
+
|
|
96
|
+
def test_fail_medium(self):
|
|
97
|
+
self.write('src/a.py', 'import legacy')
|
|
98
|
+
self.assertEqual(self.check('Never use legacy', fail_on='medium')['exit_code'], 2)
|
|
99
|
+
|
|
100
|
+
def test_no_self_match(self):
|
|
101
|
+
self.assertEqual(self.check('Never use legacy\nimport legacy')['exit_code'], 0)
|
|
102
|
+
|
|
103
|
+
def test_custom_instruction_excluded(self):
|
|
104
|
+
self.write('rules.py', 'Never use legacy\nimport legacy')
|
|
105
|
+
self.assertEqual(audit(self.root, files=['rules.py'])['exit_code'], 0)
|
|
106
|
+
|
|
107
|
+
def test_skipped_directories(self):
|
|
108
|
+
for directory in ('.git', 'node_modules', 'dist', '__pycache__', '.venv', 'venv', '.hidden'):
|
|
109
|
+
self.write(directory + '/x.py', 'import legacy')
|
|
110
|
+
self.assertEqual(self.check('Never use legacy')['exit_code'], 0)
|
|
111
|
+
|
|
112
|
+
def test_ignore(self):
|
|
113
|
+
r = self.check('scripts/old.py', ignore=['scripts/*'])
|
|
114
|
+
self.assertEqual((r['ignored'], r['exit_code']), (1, 0))
|
|
115
|
+
|
|
116
|
+
def test_dedup(self):
|
|
117
|
+
self.assertEqual(len(self.check('src/a.py src/a.py')['findings']), 1)
|
|
118
|
+
|
|
119
|
+
def test_distinct_lines(self):
|
|
120
|
+
self.assertEqual(len(self.check('src/a.py\nsrc/a.py')['findings']), 2)
|
|
121
|
+
|
|
122
|
+
def test_default_order(self):
|
|
123
|
+
for name in ('MUSE.md', 'CLAUDE.md', 'AGENTS.md', 'CONTRIBUTING.md'):
|
|
124
|
+
self.write(name, 'src/a.py')
|
|
125
|
+
self.assertEqual(audit(self.root)['files_read'], ['AGENTS.md', 'CLAUDE.md', 'MUSE.md', 'CONTRIBUTING.md'])
|
|
126
|
+
|
|
127
|
+
def test_explicit_missing(self):
|
|
128
|
+
self.assertEqual(audit(self.root, files=['absent.md'])['parse_error_count'], 1)
|
|
129
|
+
|
|
130
|
+
def test_invalid_utf8(self):
|
|
131
|
+
(self.root / 'AGENTS.md').write_bytes(b'\xff')
|
|
132
|
+
self.assertEqual(audit(self.root)['exit_code'], 2)
|
|
133
|
+
|
|
134
|
+
def test_unreadable(self):
|
|
135
|
+
self.write('AGENTS.md')
|
|
136
|
+
with patch.object(Path, 'read_text', side_effect=PermissionError('denied')):
|
|
137
|
+
r = audit(self.root)
|
|
138
|
+
self.assertEqual((r['parse_error_count'], r['exit_code']), (1, 2))
|
|
139
|
+
|
|
140
|
+
def test_cap_order(self):
|
|
141
|
+
r = self.check('\n'.join('src/missing%d.py' % n for n in range(502)))
|
|
142
|
+
self.assertEqual(len(r['findings']), 500)
|
|
143
|
+
self.assertTrue(r['truncated'])
|
|
144
|
+
self.assertEqual(r['findings'][-1]['line'], 500)
|
|
145
|
+
|
|
146
|
+
def test_exact_cap(self):
|
|
147
|
+
r = self.check('\n'.join('src/missing%d.py' % n for n in range(500)))
|
|
148
|
+
self.assertFalse(r['truncated'])
|
|
149
|
+
|
|
150
|
+
def test_realistic(self):
|
|
151
|
+
self.write('scripts/new.py')
|
|
152
|
+
self.write('checks/test_app.py')
|
|
153
|
+
self.write('src/app.py', 'import legacy')
|
|
154
|
+
r = self.check('Run `python scripts/old.py`\nTests: tests/old/\nNever use legacy')
|
|
155
|
+
self.assertEqual(r['totals']['severity'], {'HIGH': 2, 'MEDIUM': 1})
|
|
156
|
+
self.assertEqual(r['exit_code'], 2)
|
|
157
|
+
self.assertIn('verify manually', r['findings'][-1]['detail'])
|
|
158
|
+
|
|
159
|
+
def test_invalid_repo(self):
|
|
160
|
+
self.assertEqual(audit(self.root / 'missing')['exit_code'], 2)
|
|
161
|
+
|
|
162
|
+
def test_high_after_cap_still_fails(self):
|
|
163
|
+
self.write('src/a.py', 'import legacy')
|
|
164
|
+
r = self.check('Never use legacy\n' * 500 + 'src/missing.py')
|
|
165
|
+
self.assertTrue(r['truncated'])
|
|
166
|
+
self.assertEqual(r['totals']['severity']['HIGH'], 0)
|
|
167
|
+
self.assertEqual(r['exit_code'], 2)
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
from contextlib import redirect_stdout
|
|
2
|
+
import io
|
|
3
|
+
import json
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
import tempfile
|
|
6
|
+
import unittest
|
|
7
|
+
from instruction_drift.cli import main
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class CliTests(unittest.TestCase):
|
|
11
|
+
def setUp(self):
|
|
12
|
+
self.temp = tempfile.TemporaryDirectory()
|
|
13
|
+
self.addCleanup(self.temp.cleanup)
|
|
14
|
+
self.root = Path(self.temp.name)
|
|
15
|
+
|
|
16
|
+
def run_cli(self, *args):
|
|
17
|
+
out = io.StringIO()
|
|
18
|
+
with redirect_stdout(out):
|
|
19
|
+
code = main(['audit', str(self.root), *args])
|
|
20
|
+
return code, out.getvalue()
|
|
21
|
+
|
|
22
|
+
def test_empty_note(self):
|
|
23
|
+
code, out = self.run_cli()
|
|
24
|
+
self.assertEqual(code, 0)
|
|
25
|
+
self.assertIn('no instruction files', out)
|
|
26
|
+
self.assertIn('PASS', out)
|
|
27
|
+
|
|
28
|
+
def test_json(self):
|
|
29
|
+
code, out = self.run_cli('--json')
|
|
30
|
+
r = json.loads(out)
|
|
31
|
+
self.assertEqual(r['exit_code'], code)
|
|
32
|
+
self.assertEqual(r['parse_error_count'], 0)
|
|
33
|
+
|
|
34
|
+
def test_high(self):
|
|
35
|
+
(self.root / 'AGENTS.md').write_text('src/missing.py')
|
|
36
|
+
code, out = self.run_cli()
|
|
37
|
+
self.assertEqual(code, 2)
|
|
38
|
+
self.assertIn('AGENTS.md:1: HIGH', out)
|
|
39
|
+
|
|
40
|
+
def test_warn_and_threshold(self):
|
|
41
|
+
(self.root / 'AGENTS.md').write_text('Never use legacy')
|
|
42
|
+
(self.root / 'app.py').write_text('import legacy')
|
|
43
|
+
self.assertEqual(self.run_cli()[0], 1)
|
|
44
|
+
self.assertEqual(self.run_cli('--fail-on', 'medium')[0], 2)
|
|
45
|
+
|
|
46
|
+
def test_parse_error(self):
|
|
47
|
+
(self.root / 'AGENTS.md').write_bytes(b'\xff')
|
|
48
|
+
code, out = self.run_cli('--json')
|
|
49
|
+
self.assertEqual(code, 2)
|
|
50
|
+
self.assertEqual(json.loads(out)['parse_error_count'], 1)
|
|
51
|
+
|
|
52
|
+
def test_repeat_files_ignore(self):
|
|
53
|
+
(self.root / 'one.md').write_text('src/a.py')
|
|
54
|
+
(self.root / 'two.md').write_text('docs/b.md scripts/a.sh')
|
|
55
|
+
code, out = self.run_cli('--files', 'one.md', '--files', 'two.md', '--ignore', 'src/*', 'docs/*', '--ignore', 'scripts/*', '--json')
|
|
56
|
+
self.assertEqual(code, 0)
|
|
57
|
+
self.assertEqual(json.loads(out)['ignored'], 3)
|
|
58
|
+
|
|
59
|
+
def test_help(self):
|
|
60
|
+
out = io.StringIO()
|
|
61
|
+
with redirect_stdout(out), self.assertRaises(SystemExit) as context:
|
|
62
|
+
main(['--help'])
|
|
63
|
+
self.assertEqual(context.exception.code, 0)
|
|
64
|
+
self.assertIn('audit', out.getvalue())
|
|
65
|
+
|
|
66
|
+
def test_version(self):
|
|
67
|
+
out = io.StringIO()
|
|
68
|
+
with redirect_stdout(out), self.assertRaises(SystemExit) as context:
|
|
69
|
+
main(['--version'])
|
|
70
|
+
self.assertEqual(context.exception.code, 0)
|
|
71
|
+
self.assertIn('0.1.0', out.getvalue())
|
|
72
|
+
|
|
73
|
+
def test_realistic_json(self):
|
|
74
|
+
(self.root / 'AGENTS.md').write_text('`python scripts/old.py`\ntests/old/\nNever use legacy')
|
|
75
|
+
(self.root / 'app.py').write_text('import legacy')
|
|
76
|
+
(self.root / 'scripts').mkdir()
|
|
77
|
+
(self.root / 'scripts/new.py').write_text('')
|
|
78
|
+
(self.root / 'checks').mkdir()
|
|
79
|
+
code, out = self.run_cli('--json')
|
|
80
|
+
self.assertEqual(code, 2)
|
|
81
|
+
self.assertEqual(json.loads(out)['totals']['severity'], {'HIGH': 2, 'MEDIUM': 1})
|
|
82
|
+
|
|
83
|
+
def test_human_json_totals(self):
|
|
84
|
+
(self.root / 'AGENTS.md').write_text('src/a.py')
|
|
85
|
+
_, human = self.run_cli()
|
|
86
|
+
_, encoded = self.run_cli('--json')
|
|
87
|
+
self.assertIn(json.dumps(json.loads(encoded)['totals'], sort_keys=True), human)
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import unittest
|
|
2
|
+
from instruction_drift.extract import extract, path_token
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class ExtractTests(unittest.TestCase):
|
|
6
|
+
def test_bare_path(self):
|
|
7
|
+
self.assertEqual(extract('See src/app.py')[0].claim, 'src/app.py')
|
|
8
|
+
|
|
9
|
+
def test_backtick_path(self):
|
|
10
|
+
self.assertEqual(extract('See `docs/help.md`.')[0].kind, 'referenced-file')
|
|
11
|
+
|
|
12
|
+
def test_directory(self):
|
|
13
|
+
self.assertEqual(path_token('tests/old/'), 'tests/old/')
|
|
14
|
+
|
|
15
|
+
def test_url(self):
|
|
16
|
+
self.assertIsNone(path_token('https://example.org/src/a.py'))
|
|
17
|
+
|
|
18
|
+
def test_http_url(self):
|
|
19
|
+
self.assertIsNone(path_token('http://example.org/a.py'))
|
|
20
|
+
|
|
21
|
+
def test_glob(self):
|
|
22
|
+
self.assertIsNone(path_token('src/*.py'))
|
|
23
|
+
|
|
24
|
+
def test_unknown_extension(self):
|
|
25
|
+
self.assertIsNone(path_token('other/a.xyz'))
|
|
26
|
+
|
|
27
|
+
def test_all_extensions(self):
|
|
28
|
+
for ext in ('py', 'js', 'ts', 'md', 'json', 'yaml', 'yml', 'toml', 'sh', 'go', 'rs', 'txt'):
|
|
29
|
+
self.assertEqual(path_token('other/a.' + ext), 'other/a.' + ext)
|
|
30
|
+
|
|
31
|
+
def test_directory_roots(self):
|
|
32
|
+
for root in ('src', 'tests', 'docs', 'examples', 'scripts', 'bin'):
|
|
33
|
+
self.assertEqual(path_token(root + '/sub/'), root + '/sub/')
|
|
34
|
+
|
|
35
|
+
def test_command(self):
|
|
36
|
+
self.assertEqual(extract('Run `python scripts/run.py`')[0].kind, 'script-in-command')
|
|
37
|
+
|
|
38
|
+
def test_dot_script(self):
|
|
39
|
+
item = extract('Run `./scripts/run.sh`')[0]
|
|
40
|
+
self.assertEqual(item.claim, './scripts/run.sh')
|
|
41
|
+
self.assertEqual(item.kind, 'script-in-command')
|
|
42
|
+
|
|
43
|
+
def test_make(self):
|
|
44
|
+
self.assertEqual(extract('Run `make check`')[0].claim, 'make check')
|
|
45
|
+
|
|
46
|
+
def test_prohibition_forms(self):
|
|
47
|
+
for prefix in ('Do not use', 'never use', 'must not use', 'avoid using'):
|
|
48
|
+
self.assertEqual(extract(prefix + ' `legacy`.')[0].claim, 'legacy')
|
|
49
|
+
|
|
50
|
+
def test_line_number(self):
|
|
51
|
+
self.assertEqual(extract('\n\nsrc/a.py')[0].line, 3)
|
|
52
|
+
|
|
53
|
+
def test_dedup(self):
|
|
54
|
+
self.assertEqual(len(extract('src/a.py src/a.py')), 1)
|
|
55
|
+
|
|
56
|
+
def test_no_plain_filename(self):
|
|
57
|
+
self.assertEqual(extract('Read README.md'), [])
|