interactive-claim-audit 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- interactive_claim_audit-0.1.0/LICENSE +21 -0
- interactive_claim_audit-0.1.0/MANIFEST.in +3 -0
- interactive_claim_audit-0.1.0/PKG-INFO +141 -0
- interactive_claim_audit-0.1.0/README.md +131 -0
- interactive_claim_audit-0.1.0/pyproject.toml +18 -0
- interactive_claim_audit-0.1.0/setup.cfg +4 -0
- interactive_claim_audit-0.1.0/src/interactive_claim_audit/__init__.py +2 -0
- interactive_claim_audit-0.1.0/src/interactive_claim_audit/__main__.py +47 -0
- interactive_claim_audit-0.1.0/src/interactive_claim_audit/audit.py +158 -0
- interactive_claim_audit-0.1.0/src/interactive_claim_audit.egg-info/PKG-INFO +141 -0
- interactive_claim_audit-0.1.0/src/interactive_claim_audit.egg-info/SOURCES.txt +25 -0
- interactive_claim_audit-0.1.0/src/interactive_claim_audit.egg-info/dependency_links.txt +1 -0
- interactive_claim_audit-0.1.0/src/interactive_claim_audit.egg-info/entry_points.txt +2 -0
- interactive_claim_audit-0.1.0/src/interactive_claim_audit.egg-info/top_level.txt +1 -0
- interactive_claim_audit-0.1.0/tests/fixtures/bad/interactive/README.md +1 -0
- interactive_claim_audit-0.1.0/tests/fixtures/bad/interactive/main.py +1 -0
- interactive_claim_audit-0.1.0/tests/fixtures/bad/noninteractive/README.md +1 -0
- interactive_claim_audit-0.1.0/tests/fixtures/bad/noninteractive/main.py +1 -0
- interactive_claim_audit-0.1.0/tests/fixtures/good/interactive/README.md +1 -0
- interactive_claim_audit-0.1.0/tests/fixtures/good/interactive/main.py +1 -0
- interactive_claim_audit-0.1.0/tests/fixtures/good/noninteractive/README.md +1 -0
- interactive_claim_audit-0.1.0/tests/fixtures/good/noninteractive/main.py +1 -0
- interactive_claim_audit-0.1.0/tests/fixtures/testonly/.env +1 -0
- interactive_claim_audit-0.1.0/tests/fixtures/testonly/README.md +1 -0
- interactive_claim_audit-0.1.0/tests/fixtures/testonly/main.py +1 -0
- interactive_claim_audit-0.1.0/tests/fixtures/testonly/tests/test_work.py +2 -0
- interactive_claim_audit-0.1.0/tests/test_audit.py +258 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 interactive-claim-audit contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: interactive-claim-audit
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Offline static audit of documented interactivity claims.
|
|
5
|
+
License: MIT
|
|
6
|
+
Requires-Python: >=3.9
|
|
7
|
+
Description-Content-Type: text/markdown
|
|
8
|
+
License-File: LICENSE
|
|
9
|
+
Dynamic: license-file
|
|
10
|
+
|
|
11
|
+
# interactive-claim-audit
|
|
12
|
+
|
|
13
|
+
Offline static audit of documented interactivity claims.
|
|
14
|
+
|
|
15
|
+
Version 0.1.0 · Python 3.9+ · MIT · standard library only at runtime.
|
|
16
|
+
|
|
17
|
+
Compare documentation promises about human interaction with named Python call sites. The auditor parses source without importing or executing the audited project, uses no network, skips symlinks, and skips `.env` and `.env.*` file contents.
|
|
18
|
+
|
|
19
|
+
## Usage
|
|
20
|
+
|
|
21
|
+
Install from a local checkout:
|
|
22
|
+
|
|
23
|
+
```sh
|
|
24
|
+
python3 -m pip install .
|
|
25
|
+
interactive-claim-audit /path/to/project
|
|
26
|
+
python3 -m interactive_claim_audit /path/to/project --json
|
|
27
|
+
interactive-claim-audit /path/to/project --verbose --ignore 'vendor/*'
|
|
28
|
+
interactive-claim-audit /path/to/project --fail-on HIGH
|
|
29
|
+
interactive-claim-audit --version
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
The path defaults to the current directory. `--ignore GLOB` is repeatable and matches relative paths. `--verbose` adds verified families and parse warnings to text output. `--json` emits a JSON object containing `status`, `findings`, `verified_claims`, `claims`, `production_evidence`, `test_only_evidence`, and `errors`.
|
|
33
|
+
|
|
34
|
+
Exit codes are 0 for PASS, 2 for findings meeting the selected threshold, and 1 for usage errors. The default threshold is MEDIUM; `--fail-on HIGH` leaves MEDIUM findings visible but permits exit 0.
|
|
35
|
+
|
|
36
|
+
## Rules
|
|
37
|
+
|
|
38
|
+
Documentation occurrences collapse into two families, `noninteractive` and `interactive`, evaluated independently. Each failing family produces one finding containing all applicable evidence items.
|
|
39
|
+
|
|
40
|
+
| Rule | Severity | Condition |
|
|
41
|
+
| --- | --- | --- |
|
|
42
|
+
| `documented-noninteractive-claim-false` | HIGH | The noninteractive family has production evidence. |
|
|
43
|
+
| `documented-noninteractive-claim-false-test-only` | MEDIUM | That family has evidence only in test or fixture sources. |
|
|
44
|
+
| `documented-interactive-claim-false` | HIGH | The interactive family lacks production evidence and has no test evidence. |
|
|
45
|
+
| `documented-interactive-claim-false-test-only` | MEDIUM | That family has evidence only in test or fixture sources. |
|
|
46
|
+
|
|
47
|
+
A claimed family with no production Python files produces one HIGH finding with `no-source` evidence, taking priority over the test-only rules. With production files present, the noninteractive family is verified when evidence is absent everywhere; the interactive family is verified when production evidence exists. Verified families appear in `verified_claims`. A tree with zero documentation claims passes regardless of its call sites.
|
|
48
|
+
|
|
49
|
+
## Claim vocabulary
|
|
50
|
+
|
|
51
|
+
Only root `README.md` and `docs/**/*.md` supply claims. Fenced code blocks are excluded; inline code spans count as prose. Matching is case-insensitive and limited to the exact phrases below; hyphen and space spellings are accepted only where explicitly listed.
|
|
52
|
+
|
|
53
|
+
```text
|
|
54
|
+
NONINTERACTIVE
|
|
55
|
+
non-interactive
|
|
56
|
+
non interactive
|
|
57
|
+
not interactive
|
|
58
|
+
no prompts
|
|
59
|
+
no user input
|
|
60
|
+
without user input
|
|
61
|
+
does not prompt
|
|
62
|
+
doesn't prompt
|
|
63
|
+
never prompts
|
|
64
|
+
without prompting
|
|
65
|
+
no interaction required
|
|
66
|
+
no confirmation prompt
|
|
67
|
+
no manual intervention
|
|
68
|
+
does not ask
|
|
69
|
+
never asks
|
|
70
|
+
runs unattended
|
|
71
|
+
unattended
|
|
72
|
+
CI-friendly
|
|
73
|
+
CI friendly
|
|
74
|
+
suitable for CI
|
|
75
|
+
works in CI
|
|
76
|
+
|
|
77
|
+
INTERACTIVE
|
|
78
|
+
interactive mode
|
|
79
|
+
interactive prompt
|
|
80
|
+
interactive use
|
|
81
|
+
interactive confirmation
|
|
82
|
+
confirmation prompt
|
|
83
|
+
prompts for confirmation
|
|
84
|
+
asks for confirmation
|
|
85
|
+
asks the user
|
|
86
|
+
prompts the user
|
|
87
|
+
prompt for confirmation
|
|
88
|
+
ask the user
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
Negation is asymmetric: a sentence containing `no`, `not`, `never`, `without`, `doesn't`, `don't`, `isn't`, or `does not` suppresses the interactive family. Matches in the noninteractive family still count, including its negated phrases. Overlap masking finds the noninteractive family first, masks its matched spans, and searches the remaining text for the interactive family. This prevents a substring from generating an extra family match.
|
|
92
|
+
|
|
93
|
+
The bare words “interactive” and “prompt” are not claims. Paraphrases outside the vocabulary are not inferred. Contradictory documentation can supply both families; each receives its own evaluation.
|
|
94
|
+
|
|
95
|
+
## Named evidence table
|
|
96
|
+
|
|
97
|
+
Evidence is collected from Python AST calls and deduplicated by relative file, line, and kind. Imports and aliases are tracked per file where indicated.
|
|
98
|
+
|
|
99
|
+
| Named call | Evidence treatment |
|
|
100
|
+
| --- | --- |
|
|
101
|
+
| `input(...)`, `builtins.input(...)` | Counts, including a locally shadowed `input`. |
|
|
102
|
+
| `.getpass(...)` | Counts on any receiver spelling. |
|
|
103
|
+
| Imported bare `getpass(...)` | Counts with `from getpass import getpass`, including aliases; a bare name without that import does not count. |
|
|
104
|
+
| `.prompt(...)` | Counts on any receiver, including Click, Questionary objects, and toolkit sessions. |
|
|
105
|
+
| `.confirm(...)` | Counts on any receiver. |
|
|
106
|
+
| `questionary.<name>(...)` or imported Questionary functions | Counts with tracked module or function imports and aliases. |
|
|
107
|
+
| Toolkit `prompt(...)` | Counts with a per-file `prompt_toolkit` import; imported `prompt` aliases and `shortcuts` imports are recognized. |
|
|
108
|
+
|
|
109
|
+
Production evidence excludes `tests/`, `fixtures/`, test filename patterns (`test_*.py`, `*_test.py`), `docs/`, and `examples/`. Test and fixture evidence is collected separately; docs and examples supply no source evidence. Symlinked files and directories are skipped. Parse and decoding failures are reported in `errors`; they are not proof that a file lacks relevant behavior.
|
|
110
|
+
|
|
111
|
+
## Differentiation
|
|
112
|
+
|
|
113
|
+
| Neighbour | Its question | This tool's distinction |
|
|
114
|
+
| --- | --- | --- |
|
|
115
|
+
| auth-claim-audit | Are credentials required? | Examines dependence on a human, including filename and yes/no questions. `getpass` overlaps explicitly: login evidence there, human-input evidence here. |
|
|
116
|
+
| error-claim-audit | Which failures occur? | Examines waiting for a person. |
|
|
117
|
+
| concurrency-claim-audit | What execution model is used? | Examines human interaction. |
|
|
118
|
+
| timeout-claim-audit | What time bounds apply? | Examines human blocking rather than elapsed-time guarantees. |
|
|
119
|
+
| Bandit / Ruff | What code issues exist? | Compares documentation promises with source evidence. |
|
|
120
|
+
| Click / Questionary / prompt_toolkit documentation | How are input primitives used? | Checks documentation against a project's source instead of teaching those APIs. |
|
|
121
|
+
|
|
122
|
+
## Honest limitations
|
|
123
|
+
|
|
124
|
+
This is a vocabulary-and-call-site check, not a behavioral proof. Exact-phrase claims miss paraphrases. Dead code counts: a call in an uncalled function is still evidence. Reachability, branches, flag defaults, runtime configuration, and whether a person actually sees a question are not inferred. A successful result does not prove every documented execution path behaves as intended.
|
|
125
|
+
|
|
126
|
+
Receiver types are not inferred: `.prompt`, `.confirm`, and `.getpass` calls on unrelated objects or test doubles can count. A locally defined `input` also counts. Import recognition is limited to the named evidence table; dynamic imports, indirect wrappers, and arbitrary alias assignments can escape detection.
|
|
127
|
+
|
|
128
|
+
Reading `sys.stdin.read`, `readline`, or `readlines`, including redirected or piped input, is not evidence. Printing and logging are not evidence. Argparse and Click option declarations are not evidence; in particular, `click.option(..., prompt=True)` is a documented gap because it is a declaration rather than a `.prompt(...)` call.
|
|
129
|
+
|
|
130
|
+
Curses, Urwid, and Textual interfaces are outside scope. Other libraries such as inquirer, PyInquirer, and bullet are outside scope unless their calls independently match a named pattern. Evidence found only in tests or fixtures receives MEDIUM under the rules above; it does not establish production behavior. Negation and overlap masking follow the lexical rules above rather than natural-language understanding.
|
|
131
|
+
|
|
132
|
+
## Development and packaging
|
|
133
|
+
|
|
134
|
+
```sh
|
|
135
|
+
PYTHONPATH=src python3 -m unittest discover -s tests -v
|
|
136
|
+
python3 -m build
|
|
137
|
+
# If isolated build dependencies are unavailable offline:
|
|
138
|
+
python3 -m build --no-isolation
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
The build produces a wheel and source distribution in `dist/`. The source distribution includes the test fixtures through `MANIFEST.in`.
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
# interactive-claim-audit
|
|
2
|
+
|
|
3
|
+
Offline static audit of documented interactivity claims.
|
|
4
|
+
|
|
5
|
+
Version 0.1.0 · Python 3.9+ · MIT · standard library only at runtime.
|
|
6
|
+
|
|
7
|
+
Compare documentation promises about human interaction with named Python call sites. The auditor parses source without importing or executing the audited project, uses no network, skips symlinks, and skips `.env` and `.env.*` file contents.
|
|
8
|
+
|
|
9
|
+
## Usage
|
|
10
|
+
|
|
11
|
+
Install from a local checkout:
|
|
12
|
+
|
|
13
|
+
```sh
|
|
14
|
+
python3 -m pip install .
|
|
15
|
+
interactive-claim-audit /path/to/project
|
|
16
|
+
python3 -m interactive_claim_audit /path/to/project --json
|
|
17
|
+
interactive-claim-audit /path/to/project --verbose --ignore 'vendor/*'
|
|
18
|
+
interactive-claim-audit /path/to/project --fail-on HIGH
|
|
19
|
+
interactive-claim-audit --version
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
The path defaults to the current directory. `--ignore GLOB` is repeatable and matches relative paths. `--verbose` adds verified families and parse warnings to text output. `--json` emits a JSON object containing `status`, `findings`, `verified_claims`, `claims`, `production_evidence`, `test_only_evidence`, and `errors`.
|
|
23
|
+
|
|
24
|
+
Exit codes are 0 for PASS, 2 for findings meeting the selected threshold, and 1 for usage errors. The default threshold is MEDIUM; `--fail-on HIGH` leaves MEDIUM findings visible but permits exit 0.
|
|
25
|
+
|
|
26
|
+
## Rules
|
|
27
|
+
|
|
28
|
+
Documentation occurrences collapse into two families, `noninteractive` and `interactive`, evaluated independently. Each failing family produces one finding containing all applicable evidence items.
|
|
29
|
+
|
|
30
|
+
| Rule | Severity | Condition |
|
|
31
|
+
| --- | --- | --- |
|
|
32
|
+
| `documented-noninteractive-claim-false` | HIGH | The noninteractive family has production evidence. |
|
|
33
|
+
| `documented-noninteractive-claim-false-test-only` | MEDIUM | That family has evidence only in test or fixture sources. |
|
|
34
|
+
| `documented-interactive-claim-false` | HIGH | The interactive family lacks production evidence and has no test evidence. |
|
|
35
|
+
| `documented-interactive-claim-false-test-only` | MEDIUM | That family has evidence only in test or fixture sources. |
|
|
36
|
+
|
|
37
|
+
A claimed family with no production Python files produces one HIGH finding with `no-source` evidence, taking priority over the test-only rules. With production files present, the noninteractive family is verified when evidence is absent everywhere; the interactive family is verified when production evidence exists. Verified families appear in `verified_claims`. A tree with zero documentation claims passes regardless of its call sites.
|
|
38
|
+
|
|
39
|
+
## Claim vocabulary
|
|
40
|
+
|
|
41
|
+
Only root `README.md` and `docs/**/*.md` supply claims. Fenced code blocks are excluded; inline code spans count as prose. Matching is case-insensitive and limited to the exact phrases below; hyphen and space spellings are accepted only where explicitly listed.
|
|
42
|
+
|
|
43
|
+
```text
|
|
44
|
+
NONINTERACTIVE
|
|
45
|
+
non-interactive
|
|
46
|
+
non interactive
|
|
47
|
+
not interactive
|
|
48
|
+
no prompts
|
|
49
|
+
no user input
|
|
50
|
+
without user input
|
|
51
|
+
does not prompt
|
|
52
|
+
doesn't prompt
|
|
53
|
+
never prompts
|
|
54
|
+
without prompting
|
|
55
|
+
no interaction required
|
|
56
|
+
no confirmation prompt
|
|
57
|
+
no manual intervention
|
|
58
|
+
does not ask
|
|
59
|
+
never asks
|
|
60
|
+
runs unattended
|
|
61
|
+
unattended
|
|
62
|
+
CI-friendly
|
|
63
|
+
CI friendly
|
|
64
|
+
suitable for CI
|
|
65
|
+
works in CI
|
|
66
|
+
|
|
67
|
+
INTERACTIVE
|
|
68
|
+
interactive mode
|
|
69
|
+
interactive prompt
|
|
70
|
+
interactive use
|
|
71
|
+
interactive confirmation
|
|
72
|
+
confirmation prompt
|
|
73
|
+
prompts for confirmation
|
|
74
|
+
asks for confirmation
|
|
75
|
+
asks the user
|
|
76
|
+
prompts the user
|
|
77
|
+
prompt for confirmation
|
|
78
|
+
ask the user
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
Negation is asymmetric: a sentence containing `no`, `not`, `never`, `without`, `doesn't`, `don't`, `isn't`, or `does not` suppresses the interactive family. Matches in the noninteractive family still count, including its negated phrases. Overlap masking finds the noninteractive family first, masks its matched spans, and searches the remaining text for the interactive family. This prevents a substring from generating an extra family match.
|
|
82
|
+
|
|
83
|
+
The bare words “interactive” and “prompt” are not claims. Paraphrases outside the vocabulary are not inferred. Contradictory documentation can supply both families; each receives its own evaluation.
|
|
84
|
+
|
|
85
|
+
## Named evidence table
|
|
86
|
+
|
|
87
|
+
Evidence is collected from Python AST calls and deduplicated by relative file, line, and kind. Imports and aliases are tracked per file where indicated.
|
|
88
|
+
|
|
89
|
+
| Named call | Evidence treatment |
|
|
90
|
+
| --- | --- |
|
|
91
|
+
| `input(...)`, `builtins.input(...)` | Counts, including a locally shadowed `input`. |
|
|
92
|
+
| `.getpass(...)` | Counts on any receiver spelling. |
|
|
93
|
+
| Imported bare `getpass(...)` | Counts with `from getpass import getpass`, including aliases; a bare name without that import does not count. |
|
|
94
|
+
| `.prompt(...)` | Counts on any receiver, including Click, Questionary objects, and toolkit sessions. |
|
|
95
|
+
| `.confirm(...)` | Counts on any receiver. |
|
|
96
|
+
| `questionary.<name>(...)` or imported Questionary functions | Counts with tracked module or function imports and aliases. |
|
|
97
|
+
| Toolkit `prompt(...)` | Counts with a per-file `prompt_toolkit` import; imported `prompt` aliases and `shortcuts` imports are recognized. |
|
|
98
|
+
|
|
99
|
+
Production evidence excludes `tests/`, `fixtures/`, test filename patterns (`test_*.py`, `*_test.py`), `docs/`, and `examples/`. Test and fixture evidence is collected separately; docs and examples supply no source evidence. Symlinked files and directories are skipped. Parse and decoding failures are reported in `errors`; they are not proof that a file lacks relevant behavior.
|
|
100
|
+
|
|
101
|
+
## Differentiation
|
|
102
|
+
|
|
103
|
+
| Neighbour | Its question | This tool's distinction |
|
|
104
|
+
| --- | --- | --- |
|
|
105
|
+
| auth-claim-audit | Are credentials required? | Examines dependence on a human, including filename and yes/no questions. `getpass` overlaps explicitly: login evidence there, human-input evidence here. |
|
|
106
|
+
| error-claim-audit | Which failures occur? | Examines waiting for a person. |
|
|
107
|
+
| concurrency-claim-audit | What execution model is used? | Examines human interaction. |
|
|
108
|
+
| timeout-claim-audit | What time bounds apply? | Examines human blocking rather than elapsed-time guarantees. |
|
|
109
|
+
| Bandit / Ruff | What code issues exist? | Compares documentation promises with source evidence. |
|
|
110
|
+
| Click / Questionary / prompt_toolkit documentation | How are input primitives used? | Checks documentation against a project's source instead of teaching those APIs. |
|
|
111
|
+
|
|
112
|
+
## Honest limitations
|
|
113
|
+
|
|
114
|
+
This is a vocabulary-and-call-site check, not a behavioral proof. Exact-phrase claims miss paraphrases. Dead code counts: a call in an uncalled function is still evidence. Reachability, branches, flag defaults, runtime configuration, and whether a person actually sees a question are not inferred. A successful result does not prove every documented execution path behaves as intended.
|
|
115
|
+
|
|
116
|
+
Receiver types are not inferred: `.prompt`, `.confirm`, and `.getpass` calls on unrelated objects or test doubles can count. A locally defined `input` also counts. Import recognition is limited to the named evidence table; dynamic imports, indirect wrappers, and arbitrary alias assignments can escape detection.
|
|
117
|
+
|
|
118
|
+
Reading `sys.stdin.read`, `readline`, or `readlines`, including redirected or piped input, is not evidence. Printing and logging are not evidence. Argparse and Click option declarations are not evidence; in particular, `click.option(..., prompt=True)` is a documented gap because it is a declaration rather than a `.prompt(...)` call.
|
|
119
|
+
|
|
120
|
+
Curses, Urwid, and Textual interfaces are outside scope. Other libraries such as inquirer, PyInquirer, and bullet are outside scope unless their calls independently match a named pattern. Evidence found only in tests or fixtures receives MEDIUM under the rules above; it does not establish production behavior. Negation and overlap masking follow the lexical rules above rather than natural-language understanding.
|
|
121
|
+
|
|
122
|
+
## Development and packaging
|
|
123
|
+
|
|
124
|
+
```sh
|
|
125
|
+
PYTHONPATH=src python3 -m unittest discover -s tests -v
|
|
126
|
+
python3 -m build
|
|
127
|
+
# If isolated build dependencies are unavailable offline:
|
|
128
|
+
python3 -m build --no-isolation
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
The build produces a wheel and source distribution in `dist/`. The source distribution includes the test fixtures through `MANIFEST.in`.
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=61"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "interactive-claim-audit"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Offline static audit of documented interactivity claims."
|
|
9
|
+
requires-python = ">=3.9"
|
|
10
|
+
license = {text = "MIT"}
|
|
11
|
+
readme = "README.md"
|
|
12
|
+
dependencies = []
|
|
13
|
+
|
|
14
|
+
[project.scripts]
|
|
15
|
+
interactive-claim-audit = "interactive_claim_audit.__main__:main"
|
|
16
|
+
|
|
17
|
+
[tool.setuptools.packages.find]
|
|
18
|
+
where = ["src"]
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
"""Command line entry point."""
|
|
2
|
+
import argparse
|
|
3
|
+
import json
|
|
4
|
+
import sys
|
|
5
|
+
from . import __version__
|
|
6
|
+
from .audit import audit
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class Parser(argparse.ArgumentParser):
|
|
10
|
+
def error(self, message):
|
|
11
|
+
self.print_usage(sys.stderr)
|
|
12
|
+
self.exit(1, self.prog + ': error: ' + message + '\n')
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def main(argv=None):
|
|
16
|
+
parser = Parser(description='Offline static audit of documented interactivity claims.')
|
|
17
|
+
parser.add_argument('path', nargs='?', default='.')
|
|
18
|
+
parser.add_argument('--json', action='store_true', dest='as_json')
|
|
19
|
+
parser.add_argument('--verbose', action='store_true')
|
|
20
|
+
parser.add_argument('--ignore', action='append', default=[], metavar='GLOB')
|
|
21
|
+
parser.add_argument('--fail-on', choices=['MEDIUM', 'HIGH'], default='MEDIUM')
|
|
22
|
+
parser.add_argument('--version', action='version', version=__version__)
|
|
23
|
+
args = parser.parse_args(argv)
|
|
24
|
+
try:
|
|
25
|
+
result = audit(args.path, args.ignore)
|
|
26
|
+
except (OSError, ValueError) as exc:
|
|
27
|
+
parser.error(str(exc))
|
|
28
|
+
failed = any(f['severity'] == 'HIGH' or args.fail_on == 'MEDIUM' for f in result['findings'])
|
|
29
|
+
result['status'] = 'FAIL' if failed else 'PASS'
|
|
30
|
+
if args.as_json:
|
|
31
|
+
print(json.dumps(result, indent=2))
|
|
32
|
+
else:
|
|
33
|
+
print(result['status'])
|
|
34
|
+
for finding in result['findings']:
|
|
35
|
+
print(finding['severity'] + ' ' + finding['rule'])
|
|
36
|
+
for item in finding['evidence']:
|
|
37
|
+
print(' ' + item.get('path', '') + ':' + str(item.get('line', '')) + ' ' + item['kind'])
|
|
38
|
+
if args.verbose:
|
|
39
|
+
for claim in result['verified_claims']:
|
|
40
|
+
print('verified: ' + claim['family'])
|
|
41
|
+
for error in result['errors']:
|
|
42
|
+
print('parse warning: ' + error['path'] + ': ' + error['message'])
|
|
43
|
+
return 2 if failed else 0
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
if __name__ == '__main__':
|
|
47
|
+
sys.exit(main())
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
"""Static documentation and prompt-call auditing; never imports audited code."""
|
|
2
|
+
import ast
|
|
3
|
+
import fnmatch
|
|
4
|
+
import os
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
import re
|
|
7
|
+
|
|
8
|
+
NONINTERACTIVE = (
|
|
9
|
+
'non-interactive', 'non interactive', 'not interactive', 'no prompts',
|
|
10
|
+
'no user input', 'without user input', 'does not prompt', "doesn't prompt",
|
|
11
|
+
'never prompts', 'without prompting', 'no interaction required',
|
|
12
|
+
'no confirmation prompt', 'no manual intervention', 'does not ask',
|
|
13
|
+
'never asks', 'runs unattended', 'unattended', 'CI-friendly', 'CI friendly',
|
|
14
|
+
'suitable for CI', 'works in CI',
|
|
15
|
+
)
|
|
16
|
+
INTERACTIVE = (
|
|
17
|
+
'interactive mode', 'interactive prompt', 'interactive use',
|
|
18
|
+
'interactive confirmation', 'confirmation prompt', 'prompts for confirmation',
|
|
19
|
+
'asks for confirmation', 'asks the user', 'prompts the user',
|
|
20
|
+
'prompt for confirmation', 'ask the user',
|
|
21
|
+
)
|
|
22
|
+
NEGATION = re.compile(r"\b(?:no|not|never|without|doesn't|don't|isn't|does not)\b", re.I)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def phrase_pattern(phrases):
|
|
26
|
+
return re.compile(r'(?<!\w)(?:' + '|'.join(re.escape(p) for p in sorted(phrases, key=len, reverse=True)) + r')(?!\w)', re.I)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def extract_claims(text, path='README.md'):
|
|
30
|
+
"""Return phrase occurrences with locations, excluding Markdown fences."""
|
|
31
|
+
lines = []
|
|
32
|
+
fence = None
|
|
33
|
+
for line in text.splitlines(keepends=True):
|
|
34
|
+
match = re.match(r'^\s{0,3}(`{3,}|~{3,})', line)
|
|
35
|
+
if fence:
|
|
36
|
+
lines.append('\n' if line.endswith('\n') else '')
|
|
37
|
+
if match and match[1][0] == fence[0] and len(match[1]) >= len(fence) and not line[match.end():].strip():
|
|
38
|
+
fence = None
|
|
39
|
+
elif match:
|
|
40
|
+
fence = match[1]
|
|
41
|
+
lines.append('\n' if line.endswith('\n') else '')
|
|
42
|
+
else:
|
|
43
|
+
lines.append(line)
|
|
44
|
+
prose = ''.join(lines)
|
|
45
|
+
result = []
|
|
46
|
+
for sentence in re.finditer(r'[^.!?]+(?:[.!?]|$)', prose):
|
|
47
|
+
value = sentence[0]
|
|
48
|
+
masked = list(value)
|
|
49
|
+
for match in phrase_pattern(NONINTERACTIVE).finditer(value):
|
|
50
|
+
result.append({'family': 'noninteractive', 'path': path,
|
|
51
|
+
'line': prose.count('\n', 0, sentence.start() + match.start()) + 1,
|
|
52
|
+
'phrase': match[0]})
|
|
53
|
+
masked[match.start():match.end()] = ' ' * len(match[0])
|
|
54
|
+
if not NEGATION.search(value):
|
|
55
|
+
for match in phrase_pattern(INTERACTIVE).finditer(''.join(masked)):
|
|
56
|
+
result.append({'family': 'interactive', 'path': path,
|
|
57
|
+
'line': prose.count('\n', 0, sentence.start() + match.start()) + 1,
|
|
58
|
+
'phrase': match[0]})
|
|
59
|
+
return result
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def extract_evidence(source, path):
|
|
63
|
+
tree = ast.parse(source, filename=path)
|
|
64
|
+
getpass_names = set()
|
|
65
|
+
questionary_names = set()
|
|
66
|
+
questionary_functions = set()
|
|
67
|
+
toolkit_names = set()
|
|
68
|
+
toolkit_imported = False
|
|
69
|
+
for node in ast.walk(tree):
|
|
70
|
+
if isinstance(node, ast.Import):
|
|
71
|
+
for alias in node.names:
|
|
72
|
+
if alias.name == 'questionary':
|
|
73
|
+
questionary_names.add(alias.asname or alias.name)
|
|
74
|
+
if alias.name == 'prompt_toolkit' or alias.name.startswith('prompt_toolkit.'):
|
|
75
|
+
toolkit_names.add(alias.asname or alias.name.split('.')[0])
|
|
76
|
+
toolkit_imported = True
|
|
77
|
+
elif isinstance(node, ast.ImportFrom):
|
|
78
|
+
if node.module == 'getpass':
|
|
79
|
+
getpass_names.update(a.asname or a.name for a in node.names if a.name == 'getpass')
|
|
80
|
+
if node.module == 'questionary':
|
|
81
|
+
questionary_functions.update(a.asname or a.name for a in node.names)
|
|
82
|
+
if node.module and (node.module == 'prompt_toolkit' or node.module.startswith('prompt_toolkit.')):
|
|
83
|
+
toolkit_imported = True
|
|
84
|
+
toolkit_names.update(a.asname or a.name for a in node.names if a.name == 'shortcuts')
|
|
85
|
+
toolkit_names.update(a.asname or a.name for a in node.names if a.name == 'prompt')
|
|
86
|
+
result = {}
|
|
87
|
+
for node in ast.walk(tree):
|
|
88
|
+
if not isinstance(node, ast.Call):
|
|
89
|
+
continue
|
|
90
|
+
func = node.func
|
|
91
|
+
kind = None
|
|
92
|
+
if isinstance(func, ast.Name):
|
|
93
|
+
if func.id == 'input':
|
|
94
|
+
kind = 'input'
|
|
95
|
+
elif func.id in getpass_names:
|
|
96
|
+
kind = 'getpass'
|
|
97
|
+
elif func.id in questionary_functions:
|
|
98
|
+
kind = 'questionary'
|
|
99
|
+
elif toolkit_imported and (func.id == 'prompt' or func.id in toolkit_names and func.id != 'shortcuts'):
|
|
100
|
+
kind = 'prompt_toolkit'
|
|
101
|
+
elif isinstance(func, ast.Attribute):
|
|
102
|
+
if func.attr in ('prompt', 'confirm', 'getpass'):
|
|
103
|
+
kind = func.attr
|
|
104
|
+
elif func.attr == 'input' and isinstance(func.value, ast.Name) and func.value.id == 'builtins':
|
|
105
|
+
kind = 'input'
|
|
106
|
+
elif isinstance(func.value, ast.Name) and func.value.id in questionary_names:
|
|
107
|
+
kind = 'questionary'
|
|
108
|
+
if kind:
|
|
109
|
+
result[(path, node.lineno, kind)] = {'path': path, 'line': node.lineno, 'kind': kind}
|
|
110
|
+
return sorted(result.values(), key=lambda e: (e['path'], e['line'], e['kind']))
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def audit(root, ignore=()):
|
|
114
|
+
root = Path(root)
|
|
115
|
+
if not root.is_dir():
|
|
116
|
+
raise ValueError('audit path must be a directory')
|
|
117
|
+
claims, production, testonly, errors = [], [], [], []
|
|
118
|
+
source_count = 0
|
|
119
|
+
for directory, dirs, files in os.walk(root, followlinks=False):
|
|
120
|
+
dirs[:] = sorted(d for d in dirs if not (Path(directory) / d).is_symlink() and not any(fnmatch.fnmatch((Path(directory) / d).relative_to(root).as_posix(), p) for p in ignore))
|
|
121
|
+
for name in sorted(files):
|
|
122
|
+
file = Path(directory) / name
|
|
123
|
+
relative = file.relative_to(root)
|
|
124
|
+
path = relative.as_posix()
|
|
125
|
+
if file.is_symlink() or name == '.env' or name.startswith('.env.') or any(fnmatch.fnmatch(path, p) for p in ignore):
|
|
126
|
+
continue
|
|
127
|
+
is_doc = path == 'README.md' or (relative.parts[0] == 'docs' and file.suffix == '.md')
|
|
128
|
+
if is_doc:
|
|
129
|
+
claims.extend(extract_claims(file.read_text(encoding='utf-8'), path))
|
|
130
|
+
if file.suffix != '.py' or any(p in ('docs', 'examples') for p in relative.parts[:-1]):
|
|
131
|
+
continue
|
|
132
|
+
test = any(p in ('tests', 'fixtures') for p in relative.parts[:-1]) or name.startswith('test_') or name.endswith('_test.py')
|
|
133
|
+
if not test:
|
|
134
|
+
source_count += 1
|
|
135
|
+
try:
|
|
136
|
+
evidence = extract_evidence(file.read_text(encoding='utf-8-sig'), path)
|
|
137
|
+
except (SyntaxError, UnicodeError) as exc:
|
|
138
|
+
errors.append({'path': path, 'message': str(exc)})
|
|
139
|
+
continue
|
|
140
|
+
(testonly if test else production).extend(evidence)
|
|
141
|
+
findings, verified = [], []
|
|
142
|
+
for family in ('noninteractive', 'interactive'):
|
|
143
|
+
family_claims = [c for c in claims if c['family'] == family]
|
|
144
|
+
if not family_claims:
|
|
145
|
+
continue
|
|
146
|
+
evidence = production
|
|
147
|
+
severity, suffix = 'HIGH', ''
|
|
148
|
+
if not source_count:
|
|
149
|
+
evidence = [{'kind': 'no-source'}]
|
|
150
|
+
elif not production and testonly:
|
|
151
|
+
severity, suffix, evidence = 'MEDIUM', '-test-only', testonly
|
|
152
|
+
elif (family == 'noninteractive' and not production) or (family == 'interactive' and production):
|
|
153
|
+
verified.append({'family': family, 'claims': family_claims, 'evidence': production})
|
|
154
|
+
continue
|
|
155
|
+
findings.append({'rule': 'documented-' + family + '-claim-false' + suffix,
|
|
156
|
+
'severity': severity, 'claims': family_claims, 'evidence': evidence})
|
|
157
|
+
return {'findings': findings, 'verified_claims': verified, 'claims': claims,
|
|
158
|
+
'production_evidence': production, 'test_only_evidence': testonly, 'errors': errors}
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: interactive-claim-audit
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Offline static audit of documented interactivity claims.
|
|
5
|
+
License: MIT
|
|
6
|
+
Requires-Python: >=3.9
|
|
7
|
+
Description-Content-Type: text/markdown
|
|
8
|
+
License-File: LICENSE
|
|
9
|
+
Dynamic: license-file
|
|
10
|
+
|
|
11
|
+
# interactive-claim-audit
|
|
12
|
+
|
|
13
|
+
Offline static audit of documented interactivity claims.
|
|
14
|
+
|
|
15
|
+
Version 0.1.0 · Python 3.9+ · MIT · standard library only at runtime.
|
|
16
|
+
|
|
17
|
+
Compare documentation promises about human interaction with named Python call sites. The auditor parses source without importing or executing the audited project, uses no network, skips symlinks, and skips `.env` and `.env.*` file contents.
|
|
18
|
+
|
|
19
|
+
## Usage
|
|
20
|
+
|
|
21
|
+
Install from a local checkout:
|
|
22
|
+
|
|
23
|
+
```sh
|
|
24
|
+
python3 -m pip install .
|
|
25
|
+
interactive-claim-audit /path/to/project
|
|
26
|
+
python3 -m interactive_claim_audit /path/to/project --json
|
|
27
|
+
interactive-claim-audit /path/to/project --verbose --ignore 'vendor/*'
|
|
28
|
+
interactive-claim-audit /path/to/project --fail-on HIGH
|
|
29
|
+
interactive-claim-audit --version
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
The path defaults to the current directory. `--ignore GLOB` is repeatable and matches relative paths. `--verbose` adds verified families and parse warnings to text output. `--json` emits a JSON object containing `status`, `findings`, `verified_claims`, `claims`, `production_evidence`, `test_only_evidence`, and `errors`.
|
|
33
|
+
|
|
34
|
+
Exit codes are 0 for PASS, 2 for findings meeting the selected threshold, and 1 for usage errors. The default threshold is MEDIUM; `--fail-on HIGH` leaves MEDIUM findings visible but permits exit 0.
|
|
35
|
+
|
|
36
|
+
## Rules
|
|
37
|
+
|
|
38
|
+
Documentation occurrences collapse into two families, `noninteractive` and `interactive`, evaluated independently. Each failing family produces one finding containing all applicable evidence items.
|
|
39
|
+
|
|
40
|
+
| Rule | Severity | Condition |
|
|
41
|
+
| --- | --- | --- |
|
|
42
|
+
| `documented-noninteractive-claim-false` | HIGH | The noninteractive family has production evidence. |
|
|
43
|
+
| `documented-noninteractive-claim-false-test-only` | MEDIUM | That family has evidence only in test or fixture sources. |
|
|
44
|
+
| `documented-interactive-claim-false` | HIGH | The interactive family lacks production evidence and has no test evidence. |
|
|
45
|
+
| `documented-interactive-claim-false-test-only` | MEDIUM | That family has evidence only in test or fixture sources. |
|
|
46
|
+
|
|
47
|
+
A claimed family with no production Python files produces one HIGH finding with `no-source` evidence, taking priority over the test-only rules. With production files present, the noninteractive family is verified when evidence is absent everywhere; the interactive family is verified when production evidence exists. Verified families appear in `verified_claims`. A tree with zero documentation claims passes regardless of its call sites.
|
|
48
|
+
|
|
49
|
+
## Claim vocabulary
|
|
50
|
+
|
|
51
|
+
Only root `README.md` and `docs/**/*.md` supply claims. Fenced code blocks are excluded; inline code spans count as prose. Matching is case-insensitive and limited to the exact phrases below; hyphen and space spellings are accepted only where explicitly listed.
|
|
52
|
+
|
|
53
|
+
```text
|
|
54
|
+
NONINTERACTIVE
|
|
55
|
+
non-interactive
|
|
56
|
+
non interactive
|
|
57
|
+
not interactive
|
|
58
|
+
no prompts
|
|
59
|
+
no user input
|
|
60
|
+
without user input
|
|
61
|
+
does not prompt
|
|
62
|
+
doesn't prompt
|
|
63
|
+
never prompts
|
|
64
|
+
without prompting
|
|
65
|
+
no interaction required
|
|
66
|
+
no confirmation prompt
|
|
67
|
+
no manual intervention
|
|
68
|
+
does not ask
|
|
69
|
+
never asks
|
|
70
|
+
runs unattended
|
|
71
|
+
unattended
|
|
72
|
+
CI-friendly
|
|
73
|
+
CI friendly
|
|
74
|
+
suitable for CI
|
|
75
|
+
works in CI
|
|
76
|
+
|
|
77
|
+
INTERACTIVE
|
|
78
|
+
interactive mode
|
|
79
|
+
interactive prompt
|
|
80
|
+
interactive use
|
|
81
|
+
interactive confirmation
|
|
82
|
+
confirmation prompt
|
|
83
|
+
prompts for confirmation
|
|
84
|
+
asks for confirmation
|
|
85
|
+
asks the user
|
|
86
|
+
prompts the user
|
|
87
|
+
prompt for confirmation
|
|
88
|
+
ask the user
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
Negation is asymmetric: a sentence containing `no`, `not`, `never`, `without`, `doesn't`, `don't`, `isn't`, or `does not` suppresses the interactive family. Matches in the noninteractive family still count, including its negated phrases. Overlap masking finds the noninteractive family first, masks its matched spans, and searches the remaining text for the interactive family. This prevents a substring from generating an extra family match.
|
|
92
|
+
|
|
93
|
+
The bare words “interactive” and “prompt” are not claims. Paraphrases outside the vocabulary are not inferred. Contradictory documentation can supply both families; each receives its own evaluation.
|
|
94
|
+
|
|
95
|
+
## Named evidence table
|
|
96
|
+
|
|
97
|
+
Evidence is collected from Python AST calls and deduplicated by relative file, line, and kind. Imports and aliases are tracked per file where indicated.
|
|
98
|
+
|
|
99
|
+
| Named call | Evidence treatment |
|
|
100
|
+
| --- | --- |
|
|
101
|
+
| `input(...)`, `builtins.input(...)` | Counts, including a locally shadowed `input`. |
|
|
102
|
+
| `.getpass(...)` | Counts on any receiver spelling. |
|
|
103
|
+
| Imported bare `getpass(...)` | Counts with `from getpass import getpass`, including aliases; a bare name without that import does not count. |
|
|
104
|
+
| `.prompt(...)` | Counts on any receiver, including Click, Questionary objects, and toolkit sessions. |
|
|
105
|
+
| `.confirm(...)` | Counts on any receiver. |
|
|
106
|
+
| `questionary.<name>(...)` or imported Questionary functions | Counts with tracked module or function imports and aliases. |
|
|
107
|
+
| Toolkit `prompt(...)` | Counts with a per-file `prompt_toolkit` import; imported `prompt` aliases and `shortcuts` imports are recognized. |
|
|
108
|
+
|
|
109
|
+
Production evidence excludes `tests/`, `fixtures/`, test filename patterns (`test_*.py`, `*_test.py`), `docs/`, and `examples/`. Test and fixture evidence is collected separately; docs and examples supply no source evidence. Symlinked files and directories are skipped. Parse and decoding failures are reported in `errors`; they are not proof that a file lacks relevant behavior.
|
|
110
|
+
|
|
111
|
+
## Differentiation
|
|
112
|
+
|
|
113
|
+
| Neighbour | Its question | This tool's distinction |
|
|
114
|
+
| --- | --- | --- |
|
|
115
|
+
| auth-claim-audit | Are credentials required? | Examines dependence on a human, including filename and yes/no questions. `getpass` overlaps explicitly: login evidence there, human-input evidence here. |
|
|
116
|
+
| error-claim-audit | Which failures occur? | Examines waiting for a person. |
|
|
117
|
+
| concurrency-claim-audit | What execution model is used? | Examines human interaction. |
|
|
118
|
+
| timeout-claim-audit | What time bounds apply? | Examines human blocking rather than elapsed-time guarantees. |
|
|
119
|
+
| Bandit / Ruff | What code issues exist? | Compares documentation promises with source evidence. |
|
|
120
|
+
| Click / Questionary / prompt_toolkit documentation | How are input primitives used? | Checks documentation against a project's source instead of teaching those APIs. |
|
|
121
|
+
|
|
122
|
+
## Honest limitations
|
|
123
|
+
|
|
124
|
+
This is a vocabulary-and-call-site check, not a behavioral proof. Exact-phrase claims miss paraphrases. Dead code counts: a call in an uncalled function is still evidence. Reachability, branches, flag defaults, runtime configuration, and whether a person actually sees a question are not inferred. A successful result does not prove every documented execution path behaves as intended.
|
|
125
|
+
|
|
126
|
+
Receiver types are not inferred: `.prompt`, `.confirm`, and `.getpass` calls on unrelated objects or test doubles can count. A locally defined `input` also counts. Import recognition is limited to the named evidence table; dynamic imports, indirect wrappers, and arbitrary alias assignments can escape detection.
|
|
127
|
+
|
|
128
|
+
Reading `sys.stdin.read`, `readline`, or `readlines`, including redirected or piped input, is not evidence. Printing and logging are not evidence. Argparse and Click option declarations are not evidence; in particular, `click.option(..., prompt=True)` is a documented gap because it is a declaration rather than a `.prompt(...)` call.
|
|
129
|
+
|
|
130
|
+
Curses, Urwid, and Textual interfaces are outside scope. Other libraries such as inquirer, PyInquirer, and bullet are outside scope unless their calls independently match a named pattern. Evidence found only in tests or fixtures receives MEDIUM under the rules above; it does not establish production behavior. Negation and overlap masking follow the lexical rules above rather than natural-language understanding.
|
|
131
|
+
|
|
132
|
+
## Development and packaging
|
|
133
|
+
|
|
134
|
+
```sh
|
|
135
|
+
PYTHONPATH=src python3 -m unittest discover -s tests -v
|
|
136
|
+
python3 -m build
|
|
137
|
+
# If isolated build dependencies are unavailable offline:
|
|
138
|
+
python3 -m build --no-isolation
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
The build produces a wheel and source distribution in `dist/`. The source distribution includes the test fixtures through `MANIFEST.in`.
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
MANIFEST.in
|
|
3
|
+
README.md
|
|
4
|
+
pyproject.toml
|
|
5
|
+
src/interactive_claim_audit/__init__.py
|
|
6
|
+
src/interactive_claim_audit/__main__.py
|
|
7
|
+
src/interactive_claim_audit/audit.py
|
|
8
|
+
src/interactive_claim_audit.egg-info/PKG-INFO
|
|
9
|
+
src/interactive_claim_audit.egg-info/SOURCES.txt
|
|
10
|
+
src/interactive_claim_audit.egg-info/dependency_links.txt
|
|
11
|
+
src/interactive_claim_audit.egg-info/entry_points.txt
|
|
12
|
+
src/interactive_claim_audit.egg-info/top_level.txt
|
|
13
|
+
tests/test_audit.py
|
|
14
|
+
tests/fixtures/bad/interactive/README.md
|
|
15
|
+
tests/fixtures/bad/interactive/main.py
|
|
16
|
+
tests/fixtures/bad/noninteractive/README.md
|
|
17
|
+
tests/fixtures/bad/noninteractive/main.py
|
|
18
|
+
tests/fixtures/good/interactive/README.md
|
|
19
|
+
tests/fixtures/good/interactive/main.py
|
|
20
|
+
tests/fixtures/good/noninteractive/README.md
|
|
21
|
+
tests/fixtures/good/noninteractive/main.py
|
|
22
|
+
tests/fixtures/testonly/.env
|
|
23
|
+
tests/fixtures/testonly/README.md
|
|
24
|
+
tests/fixtures/testonly/main.py
|
|
25
|
+
tests/fixtures/testonly/tests/test_work.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
interactive_claim_audit
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
This tool prompts for confirmation.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
print("done")
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
This tool runs unattended.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
input("Continue? ")
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
This tool prompts for confirmation.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
input("Continue? ")
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
This tool runs unattended.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
print("done")
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
CLAIM_AUDIT_MARKER_ONLY_NOT_A_SECRET
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
This tool runs unattended.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
print("done")
|
|
@@ -0,0 +1,258 @@
|
|
|
1
|
+
import contextlib
|
|
2
|
+
import io
|
|
3
|
+
import json
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
import tempfile
|
|
6
|
+
import unittest
|
|
7
|
+
from unittest.mock import patch
|
|
8
|
+
from interactive_claim_audit.audit import audit, extract_claims, extract_evidence, NONINTERACTIVE, INTERACTIVE
|
|
9
|
+
from interactive_claim_audit.__main__ import main
|
|
10
|
+
|
|
11
|
+
FIXTURES = Path(__file__).parent / 'fixtures'
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class Claims(unittest.TestCase):
|
|
15
|
+
def families(self, text):
|
|
16
|
+
return {c['family'] for c in extract_claims(text)}
|
|
17
|
+
|
|
18
|
+
def test_overlap(self):
|
|
19
|
+
self.assertEqual(self.families('no confirmation prompt'), {'noninteractive'})
|
|
20
|
+
|
|
21
|
+
def test_not_interactive(self):
|
|
22
|
+
self.assertEqual(self.families('this tool is not interactive'), {'noninteractive'})
|
|
23
|
+
|
|
24
|
+
def test_not_prompt_user(self):
|
|
25
|
+
self.assertEqual(self.families('does not prompt the user'), {'noninteractive'})
|
|
26
|
+
|
|
27
|
+
def test_bare_adjective(self):
|
|
28
|
+
self.assertFalse(self.families('an interactive tool'))
|
|
29
|
+
|
|
30
|
+
def test_bare_noun(self):
|
|
31
|
+
self.assertFalse(self.families('prompt engineering'))
|
|
32
|
+
|
|
33
|
+
def test_negation(self):
|
|
34
|
+
self.assertFalse(self.families('does not run in interactive mode'))
|
|
35
|
+
|
|
36
|
+
def test_fenced(self):
|
|
37
|
+
self.assertFalse(self.families('```text\nruns unattended\n```'))
|
|
38
|
+
|
|
39
|
+
def test_tilde_fenced(self):
|
|
40
|
+
self.assertFalse(self.families('~~~~\ninteractive mode\n~~~~'))
|
|
41
|
+
|
|
42
|
+
def test_inline(self):
|
|
43
|
+
self.assertEqual(self.families('`interactive mode`'), {'interactive'})
|
|
44
|
+
|
|
45
|
+
def test_sentence_scope(self):
|
|
46
|
+
self.assertEqual(self.families('Not supported. Interactive mode.'), {'interactive'})
|
|
47
|
+
|
|
48
|
+
def test_exact_phrases(self):
|
|
49
|
+
self.assertFalse(self.families('prompts for user confirmation; runs fully unattendedly'))
|
|
50
|
+
|
|
51
|
+
def test_boundary(self):
|
|
52
|
+
self.assertFalse(self.families('xnon-interactive; interactive modes'))
|
|
53
|
+
|
|
54
|
+
def test_line_number(self):
|
|
55
|
+
self.assertEqual(extract_claims('```\nno prompts\n```\nno prompts')[0]['line'], 4)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
class Evidence(unittest.TestCase):
|
|
59
|
+
def check(self, code, count=1):
|
|
60
|
+
self.assertEqual(len(extract_evidence(code, 'main.py')), count)
|
|
61
|
+
|
|
62
|
+
def test_input(self): self.check('input()')
|
|
63
|
+
def test_builtin(self): self.check('builtins.input()')
|
|
64
|
+
def test_getpass_attribute(self): self.check('getpass.getpass()')
|
|
65
|
+
def test_any_getpass_receiver(self): self.check('obj.getpass()')
|
|
66
|
+
def test_getpass_bare_unimported(self): self.check('getpass()', 0)
|
|
67
|
+
def test_getpass_import(self): self.check('from getpass import getpass\ngetpass()')
|
|
68
|
+
def test_getpass_alias(self): self.check('from getpass import getpass as gp\ngp()')
|
|
69
|
+
def test_prompt(self): self.check('obj.prompt()')
|
|
70
|
+
def test_confirm(self): self.check('obj.confirm()')
|
|
71
|
+
def test_questionary(self): self.check('import questionary\nquestionary.text()')
|
|
72
|
+
def test_questionary_alias(self): self.check('import questionary as q\nq.text()')
|
|
73
|
+
def test_questionary_from(self): self.check('from questionary import text as t\nt()')
|
|
74
|
+
def test_questionary_unimported(self): self.check('questionary.text()', 0)
|
|
75
|
+
def test_toolkit(self): self.check('from prompt_toolkit import prompt\nprompt()')
|
|
76
|
+
def test_toolkit_alias(self): self.check('from prompt_toolkit import prompt as p\np()')
|
|
77
|
+
def test_toolkit_module(self): self.check('import prompt_toolkit\nprompt()')
|
|
78
|
+
def test_toolkit_shortcuts(self): self.check('from prompt_toolkit import shortcuts\nprompt()')
|
|
79
|
+
def test_prompt_bare(self): self.check('prompt()', 0)
|
|
80
|
+
def test_stdin(self): self.check('sys.stdin.readline()\nsys.stdin.read()\nsys.stdin.readlines()', 0)
|
|
81
|
+
def test_print(self): self.check('print("hello")', 0)
|
|
82
|
+
def test_option(self): self.check('click.option("--yes", prompt=True)', 0)
|
|
83
|
+
def test_local_input(self): self.check('def input(): pass\ninput()')
|
|
84
|
+
def test_dead_code(self): self.check('def unused():\n input()')
|
|
85
|
+
def test_dedupe(self): self.check('obj.prompt(); obj.prompt()')
|
|
86
|
+
def test_separate_lines(self): self.check('input()\ninput()', 2)
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
class Audit(unittest.TestCase):
|
|
90
|
+
def setUp(self):
|
|
91
|
+
self.temp = tempfile.TemporaryDirectory()
|
|
92
|
+
self.addCleanup(self.temp.cleanup)
|
|
93
|
+
self.root = Path(self.temp.name)
|
|
94
|
+
|
|
95
|
+
def write(self, path, text):
|
|
96
|
+
p = self.root / path
|
|
97
|
+
p.parent.mkdir(parents=True, exist_ok=True)
|
|
98
|
+
p.write_text(text, encoding='utf-8')
|
|
99
|
+
|
|
100
|
+
def test_no_claim(self):
|
|
101
|
+
self.write('main.py', 'input()')
|
|
102
|
+
self.assertFalse(audit(self.root)['findings'])
|
|
103
|
+
|
|
104
|
+
def test_ignore(self):
|
|
105
|
+
self.write('README.md', 'no prompts')
|
|
106
|
+
self.write('main.py', 'pass')
|
|
107
|
+
self.write('extra.py', 'input()')
|
|
108
|
+
self.assertFalse(audit(self.root, ['extra.py'])['findings'])
|
|
109
|
+
|
|
110
|
+
def test_both_families_no_source(self):
|
|
111
|
+
self.write('README.md', 'runs unattended. Interactive mode.')
|
|
112
|
+
findings = audit(self.root)['findings']
|
|
113
|
+
self.assertEqual(len(findings), 2)
|
|
114
|
+
self.assertTrue(all(f['severity'] == 'HIGH' and f['evidence'] == [{'kind': 'no-source'}] for f in findings))
|
|
115
|
+
|
|
116
|
+
def test_both_families_test_only(self):
|
|
117
|
+
self.write('README.md', 'runs unattended. Interactive mode.')
|
|
118
|
+
self.write('main.py', 'pass')
|
|
119
|
+
self.write('tests/test_x.py', 'input()')
|
|
120
|
+
self.assertEqual(len(audit(self.root)['findings']), 2)
|
|
121
|
+
|
|
122
|
+
def test_interactive_test_only(self):
|
|
123
|
+
self.write('README.md', 'interactive mode')
|
|
124
|
+
self.write('main.py', 'pass')
|
|
125
|
+
self.write('fixtures/x.py', 'input()')
|
|
126
|
+
self.assertEqual(audit(self.root)['findings'][0]['rule'], 'documented-interactive-claim-false-test-only')
|
|
127
|
+
|
|
128
|
+
def test_all_evidence_one_finding(self):
|
|
129
|
+
self.write('README.md', 'no prompts. runs unattended.')
|
|
130
|
+
self.write('main.py', 'input()\nobj.confirm()')
|
|
131
|
+
result = audit(self.root)
|
|
132
|
+
self.assertEqual(len(result['findings']), 1)
|
|
133
|
+
self.assertEqual(len(result['findings'][0]['evidence']), 2)
|
|
134
|
+
|
|
135
|
+
def test_docs_scope(self):
|
|
136
|
+
self.write('other.md', 'no prompts')
|
|
137
|
+
self.write('docs/nested/a.md', 'interactive mode')
|
|
138
|
+
self.write('main.py', 'input()')
|
|
139
|
+
self.assertEqual([v['family'] for v in audit(self.root)['verified_claims']], ['interactive'])
|
|
140
|
+
|
|
141
|
+
def test_excluded_sources(self):
|
|
142
|
+
self.write('README.md', 'no prompts')
|
|
143
|
+
self.write('main.py', 'pass')
|
|
144
|
+
for path in ('docs/x.py', 'examples/x.py'):
|
|
145
|
+
self.write(path, 'input()')
|
|
146
|
+
self.assertFalse(audit(self.root)['findings'])
|
|
147
|
+
|
|
148
|
+
def test_test_filename(self):
|
|
149
|
+
self.write('README.md', 'no prompts')
|
|
150
|
+
self.write('main.py', 'pass')
|
|
151
|
+
self.write('unit_test.py', 'input()')
|
|
152
|
+
self.assertEqual(audit(self.root)['findings'][0]['severity'], 'MEDIUM')
|
|
153
|
+
|
|
154
|
+
def test_symlink(self):
|
|
155
|
+
self.write('README.md', 'no prompts')
|
|
156
|
+
self.write('main.py', 'pass')
|
|
157
|
+
(self.root / 'linked.py').symlink_to(FIXTURES / 'bad/noninteractive/main.py')
|
|
158
|
+
(self.root / 'linked_dir').symlink_to(FIXTURES / 'bad/noninteractive', target_is_directory=True)
|
|
159
|
+
self.assertFalse(audit(self.root)['findings'])
|
|
160
|
+
|
|
161
|
+
def test_env_never_opened(self):
|
|
162
|
+
original = Path.open
|
|
163
|
+
def guarded(path, *args, **kwargs):
|
|
164
|
+
self.assertNotEqual(path.name, '.env')
|
|
165
|
+
return original(path, *args, **kwargs)
|
|
166
|
+
output = io.StringIO()
|
|
167
|
+
with patch.object(Path, 'open', guarded), contextlib.redirect_stdout(output):
|
|
168
|
+
self.assertEqual(main([str(FIXTURES / 'testonly'), '--json']), 2)
|
|
169
|
+
self.assertNotIn('CLAIM_AUDIT_MARKER_ONLY_NOT_A_SECRET', output.getvalue())
|
|
170
|
+
|
|
171
|
+
def test_parse_warning(self):
|
|
172
|
+
self.write('main.py', 'def !')
|
|
173
|
+
self.assertEqual(len(audit(self.root)['errors']), 1)
|
|
174
|
+
|
|
175
|
+
def test_imports_are_per_file(self):
|
|
176
|
+
self.write('README.md', 'no prompts')
|
|
177
|
+
self.write('a.py', 'from getpass import getpass')
|
|
178
|
+
self.write('b.py', 'getpass()')
|
|
179
|
+
self.assertFalse(audit(self.root)['findings'])
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
class CLI(unittest.TestCase):
|
|
183
|
+
def run_cli(self, args):
|
|
184
|
+
out = io.StringIO()
|
|
185
|
+
with contextlib.redirect_stdout(out):
|
|
186
|
+
code = main(args)
|
|
187
|
+
return code, out.getvalue()
|
|
188
|
+
|
|
189
|
+
def test_good_noninteractive(self):
|
|
190
|
+
code, out = self.run_cli([str(FIXTURES / 'good/noninteractive'), '--json'])
|
|
191
|
+
self.assertEqual(code, 0)
|
|
192
|
+
self.assertEqual(json.loads(out)['verified_claims'][0]['family'], 'noninteractive')
|
|
193
|
+
|
|
194
|
+
def test_good_interactive(self):
|
|
195
|
+
code, out = self.run_cli([str(FIXTURES / 'good/interactive'), '--json'])
|
|
196
|
+
self.assertEqual(code, 0)
|
|
197
|
+
self.assertEqual(json.loads(out)['verified_claims'][0]['family'], 'interactive')
|
|
198
|
+
|
|
199
|
+
def test_bad_noninteractive(self): self.bad('noninteractive')
|
|
200
|
+
def test_bad_interactive(self): self.bad('interactive')
|
|
201
|
+
def bad(self, family):
|
|
202
|
+
code, out = self.run_cli([str(FIXTURES / ('bad/' + family)), '--json'])
|
|
203
|
+
findings = json.loads(out)['findings']
|
|
204
|
+
self.assertEqual(code, 2)
|
|
205
|
+
self.assertEqual(len(findings), 1)
|
|
206
|
+
self.assertEqual(findings[0]['rule'], 'documented-' + family + '-claim-false')
|
|
207
|
+
self.assertEqual(findings[0]['severity'], 'HIGH')
|
|
208
|
+
|
|
209
|
+
def test_medium(self):
|
|
210
|
+
code, out = self.run_cli([str(FIXTURES / 'testonly'), '--json'])
|
|
211
|
+
self.assertEqual(code, 2)
|
|
212
|
+
findings = json.loads(out)['findings']
|
|
213
|
+
self.assertEqual(len(findings), 1)
|
|
214
|
+
self.assertEqual(findings[0]['severity'], 'MEDIUM')
|
|
215
|
+
self.assertEqual(findings[0]['rule'], 'documented-noninteractive-claim-false-test-only')
|
|
216
|
+
|
|
217
|
+
def test_threshold(self):
|
|
218
|
+
code, out = self.run_cli([str(FIXTURES / 'testonly'), '--fail-on', 'HIGH', '--json'])
|
|
219
|
+
self.assertEqual(code, 0)
|
|
220
|
+
self.assertEqual(json.loads(out)['findings'][0]['severity'], 'MEDIUM')
|
|
221
|
+
|
|
222
|
+
def test_version(self):
|
|
223
|
+
with self.assertRaises(SystemExit) as cm:
|
|
224
|
+
_, out = self.run_cli(['--version'])
|
|
225
|
+
self.assertEqual(cm.exception.code, 0)
|
|
226
|
+
output = io.StringIO()
|
|
227
|
+
with contextlib.redirect_stdout(output), self.assertRaises(SystemExit):
|
|
228
|
+
main(['--version'])
|
|
229
|
+
self.assertEqual(output.getvalue().strip(), '0.1.0')
|
|
230
|
+
|
|
231
|
+
def test_usage_error(self):
|
|
232
|
+
with contextlib.redirect_stderr(io.StringIO()), self.assertRaises(SystemExit) as cm:
|
|
233
|
+
main(['--unknown'])
|
|
234
|
+
self.assertEqual(cm.exception.code, 1)
|
|
235
|
+
|
|
236
|
+
def test_invalid_path(self):
|
|
237
|
+
with contextlib.redirect_stderr(io.StringIO()), self.assertRaises(SystemExit) as cm:
|
|
238
|
+
main(['/nonexistent/claim-audit-path'])
|
|
239
|
+
self.assertEqual(cm.exception.code, 1)
|
|
240
|
+
|
|
241
|
+
def test_verbose(self):
|
|
242
|
+
code, out = self.run_cli([str(FIXTURES / 'good/interactive'), '--verbose'])
|
|
243
|
+
self.assertEqual(code, 0)
|
|
244
|
+
self.assertIn('verified: interactive', out)
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
# Each exact phrase is independently pinned, including capitalization.
|
|
248
|
+
def phrase_test(phrase, family):
|
|
249
|
+
def test(self):
|
|
250
|
+
self.assertEqual(self.families(phrase.upper()), {family})
|
|
251
|
+
return test
|
|
252
|
+
|
|
253
|
+
for family, phrases in [('noninteractive', NONINTERACTIVE), ('interactive', INTERACTIVE)]:
|
|
254
|
+
for index, phrase in enumerate(phrases):
|
|
255
|
+
setattr(Claims, 'test_phrase_' + family + '_' + str(index), phrase_test(phrase, family))
|
|
256
|
+
|
|
257
|
+
if __name__ == '__main__':
|
|
258
|
+
unittest.main()
|