cleanup-claim-audit 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cleanup_claim_audit-0.1.0/LICENSE +21 -0
- cleanup_claim_audit-0.1.0/MANIFEST.in +2 -0
- cleanup_claim_audit-0.1.0/PKG-INFO +114 -0
- cleanup_claim_audit-0.1.0/README.md +103 -0
- cleanup_claim_audit-0.1.0/pyproject.toml +18 -0
- cleanup_claim_audit-0.1.0/setup.cfg +4 -0
- cleanup_claim_audit-0.1.0/src/cleanup_claim_audit/__init__.py +6 -0
- cleanup_claim_audit-0.1.0/src/cleanup_claim_audit/__main__.py +53 -0
- cleanup_claim_audit-0.1.0/src/cleanup_claim_audit/audit.py +278 -0
- cleanup_claim_audit-0.1.0/src/cleanup_claim_audit.egg-info/PKG-INFO +114 -0
- cleanup_claim_audit-0.1.0/src/cleanup_claim_audit.egg-info/SOURCES.txt +28 -0
- cleanup_claim_audit-0.1.0/src/cleanup_claim_audit.egg-info/dependency_links.txt +1 -0
- cleanup_claim_audit-0.1.0/src/cleanup_claim_audit.egg-info/entry_points.txt +2 -0
- cleanup_claim_audit-0.1.0/src/cleanup_claim_audit.egg-info/top_level.txt +1 -0
- cleanup_claim_audit-0.1.0/tests/fixtures/bad/cleanup/.env +1 -0
- cleanup_claim_audit-0.1.0/tests/fixtures/bad/cleanup/README.md +3 -0
- cleanup_claim_audit-0.1.0/tests/fixtures/bad/cleanup/main.py +9 -0
- cleanup_claim_audit-0.1.0/tests/fixtures/bad/nocleanup/README.md +3 -0
- cleanup_claim_audit-0.1.0/tests/fixtures/bad/nocleanup/main.py +6 -0
- cleanup_claim_audit-0.1.0/tests/fixtures/good/cleanup/README.md +3 -0
- cleanup_claim_audit-0.1.0/tests/fixtures/good/cleanup/main.py +11 -0
- cleanup_claim_audit-0.1.0/tests/fixtures/good/nocleanup/README.md +3 -0
- cleanup_claim_audit-0.1.0/tests/fixtures/good/nocleanup/main.py +9 -0
- cleanup_claim_audit-0.1.0/tests/fixtures/testonly/cleanup/README.md +3 -0
- cleanup_claim_audit-0.1.0/tests/fixtures/testonly/cleanup/main.py +9 -0
- cleanup_claim_audit-0.1.0/tests/fixtures/testonly/cleanup/tests/test_work.py +5 -0
- cleanup_claim_audit-0.1.0/tests/fixtures/testonly/nocleanup/README.md +3 -0
- cleanup_claim_audit-0.1.0/tests/fixtures/testonly/nocleanup/main.py +9 -0
- cleanup_claim_audit-0.1.0/tests/fixtures/testonly/nocleanup/tests/test_work.py +5 -0
- cleanup_claim_audit-0.1.0/tests/test_audit.py +397 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 hao li
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT, OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: cleanup-claim-audit
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Offline auditor for documented temporary-file cleanup promises in Python projects.
|
|
5
|
+
Author: hao li
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Requires-Python: >=3.9
|
|
8
|
+
Description-Content-Type: text/markdown
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Dynamic: license-file
|
|
11
|
+
|
|
12
|
+
# cleanup-claim-audit
|
|
13
|
+
|
|
14
|
+
Offline auditor for documented temporary-file cleanup promises in Python
|
|
15
|
+
projects. It checks whether a project's docs promise temp-file tidiness
|
|
16
|
+
(or promise the opposite) and whether its Python source backs that
|
|
17
|
+
promise up. It executes nothing, uses no network, and
|
|
18
|
+
never reads `.env` file contents.
|
|
19
|
+
|
|
20
|
+
## Usage
|
|
21
|
+
|
|
22
|
+
```sh
|
|
23
|
+
cleanup-claim-audit <project-root>
|
|
24
|
+
cleanup-claim-audit <project-root> --json
|
|
25
|
+
cleanup-claim-audit <project-root> --fail-on HIGH
|
|
26
|
+
cleanup-claim-audit <project-root> --ignore 'docs/draft-*'
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
Exit codes: `0` PASS, `2` FAIL, `1` usage error. `--fail-on HIGH` makes
|
|
30
|
+
MEDIUM-only results exit `0`. `--json` prints
|
|
31
|
+
`{status, findings, verified_claims, claims}`.
|
|
32
|
+
|
|
33
|
+
## Claims checked
|
|
34
|
+
|
|
35
|
+
Claims come from prose in the root `README.md` plus `docs/**/*.md`.
|
|
36
|
+
Fenced code blocks are excluded; inline code counts as prose. Claim
|
|
37
|
+
phrases are exact phrases (see below, shown in a fenced block so this
|
|
38
|
+
very README does not itself make a claim the tool would audit):
|
|
39
|
+
|
|
40
|
+
```
|
|
41
|
+
cleanup family: cleans up temporary files, cleans up temp files,
|
|
42
|
+
removes temporary files, temporary files are cleaned up,
|
|
43
|
+
no temporary files left behind, leaves no temporary files,
|
|
44
|
+
cleans up after itself, automatically cleans up temporary files
|
|
45
|
+
nocleanup family: does not clean up temporary files,
|
|
46
|
+
never cleans up temporary files, leaves temporary files behind,
|
|
47
|
+
no cleanup of temporary files, does not clean up after itself
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
A sentence in the cleanup family is suppressed when, after removing
|
|
51
|
+
the matched phrase, a negation token (`no`, `not`, `never`, `without`)
|
|
52
|
+
remains — so a sentence that only denies cleanup is claimed by the
|
|
53
|
+
nocleanup family, not by both. Bare nouns such as "temporary files" or
|
|
54
|
+
"cleanup" alone are not claims.
|
|
55
|
+
|
|
56
|
+
## Evidence
|
|
57
|
+
|
|
58
|
+
Cleanup evidence in production Python source (AST only):
|
|
59
|
+
|
|
60
|
+
- attribute calls named `unlink`, `remove`, `rmtree`, or `removedirs`
|
|
61
|
+
(any receiver: `os.unlink`, `Path.unlink`, `shutil.rmtree`, ...)
|
|
62
|
+
- `TemporaryDirectory` / `TemporaryFile` calls (self-cleaning)
|
|
63
|
+
- `atexit.register` calls (teardown registration)
|
|
64
|
+
- `NamedTemporaryFile` with the `delete` argument absent or `True`
|
|
65
|
+
(self-deleting; the default is `delete=True`)
|
|
66
|
+
|
|
67
|
+
Not evidence: bare imports alone, `mkstemp` / `mkdtemp` calls
|
|
68
|
+
(creation, not cleanup), `NamedTemporaryFile(delete=False)`, `open()`
|
|
69
|
+
writes, or anything in comments and strings.
|
|
70
|
+
|
|
71
|
+
## Rules
|
|
72
|
+
|
|
73
|
+
- `documented-cleanup-claim-false` (HIGH): a cleanup claim with zero
|
|
74
|
+
production cleanup evidence; `documented-cleanup-claim-false-test-only`
|
|
75
|
+
(MEDIUM) when the evidence exists only under `tests/` or `fixtures/`.
|
|
76
|
+
- `documented-no-cleanup-claim-false` (HIGH): a no-cleanup claim with
|
|
77
|
+
production cleanup evidence; `...-test-only` (MEDIUM) for test-only
|
|
78
|
+
evidence.
|
|
79
|
+
- No production source at all yields a HIGH finding with evidence
|
|
80
|
+
`["no-source"]` for either claimed family. No claim yields PASS.
|
|
81
|
+
|
|
82
|
+
## Differentiation
|
|
83
|
+
|
|
84
|
+
- shutdown-claim-audit asks whether a process defends its teardown
|
|
85
|
+
when interrupted (signal handlers); this asks whether stray temp
|
|
86
|
+
files get deleted during a normal run, signal or no signal.
|
|
87
|
+
- atomic-claim-audit asks whether a single write lands atomically via
|
|
88
|
+
temp file plus replace; this asks whether stray temp files get
|
|
89
|
+
deleted, not how a write lands.
|
|
90
|
+
- readonly-claim-audit asks whether files are written at all; this
|
|
91
|
+
asks whether stray temp files get deleted after being written.
|
|
92
|
+
- streaming-claim-audit asks about memory behaviour; this asks about
|
|
93
|
+
temp files on disk.
|
|
94
|
+
- bandit and ruff find code issues; they never check docs promises.
|
|
95
|
+
- The `tempfile` docs describe cleanup mechanisms at runtime; they
|
|
96
|
+
never check docs against code.
|
|
97
|
+
|
|
98
|
+
## Honest limitations
|
|
99
|
+
|
|
100
|
+
- Exact phrases, not semantics: reworded promises are missed.
|
|
101
|
+
- Dead code counts as evidence, and a cleanup call anywhere in a file
|
|
102
|
+
counts even outside a `finally` block or an `atexit` handler.
|
|
103
|
+
- Receiver types are not inferred: any `.unlink()` / `.remove()` /
|
|
104
|
+
`.rmtree()` call counts, whatever the receiver is.
|
|
105
|
+
- Attribute calls always count; bare-name calls count only when
|
|
106
|
+
imported from `tempfile` / `atexit` (aliases followed).
|
|
107
|
+
- Dynamic temp paths are not tracked, and `NamedTemporaryFile` with a
|
|
108
|
+
non-literal `delete` value is not counted as cleanup.
|
|
109
|
+
- A cleanup call is not proof the right files are removed, only that
|
|
110
|
+
a cleanup mechanism exists in the source.
|
|
111
|
+
|
|
112
|
+
## License
|
|
113
|
+
|
|
114
|
+
MIT
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
# cleanup-claim-audit
|
|
2
|
+
|
|
3
|
+
Offline auditor for documented temporary-file cleanup promises in Python
|
|
4
|
+
projects. It checks whether a project's docs promise temp-file tidiness
|
|
5
|
+
(or promise the opposite) and whether its Python source backs that
|
|
6
|
+
promise up. It executes nothing, uses no network, and
|
|
7
|
+
never reads `.env` file contents.
|
|
8
|
+
|
|
9
|
+
## Usage
|
|
10
|
+
|
|
11
|
+
```sh
|
|
12
|
+
cleanup-claim-audit <project-root>
|
|
13
|
+
cleanup-claim-audit <project-root> --json
|
|
14
|
+
cleanup-claim-audit <project-root> --fail-on HIGH
|
|
15
|
+
cleanup-claim-audit <project-root> --ignore 'docs/draft-*'
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
Exit codes: `0` PASS, `2` FAIL, `1` usage error. `--fail-on HIGH` makes
|
|
19
|
+
MEDIUM-only results exit `0`. `--json` prints
|
|
20
|
+
`{status, findings, verified_claims, claims}`.
|
|
21
|
+
|
|
22
|
+
## Claims checked
|
|
23
|
+
|
|
24
|
+
Claims come from prose in the root `README.md` plus `docs/**/*.md`.
|
|
25
|
+
Fenced code blocks are excluded; inline code counts as prose. Claim
|
|
26
|
+
phrases are exact phrases (see below, shown in a fenced block so this
|
|
27
|
+
very README does not itself make a claim the tool would audit):
|
|
28
|
+
|
|
29
|
+
```
|
|
30
|
+
cleanup family: cleans up temporary files, cleans up temp files,
|
|
31
|
+
removes temporary files, temporary files are cleaned up,
|
|
32
|
+
no temporary files left behind, leaves no temporary files,
|
|
33
|
+
cleans up after itself, automatically cleans up temporary files
|
|
34
|
+
nocleanup family: does not clean up temporary files,
|
|
35
|
+
never cleans up temporary files, leaves temporary files behind,
|
|
36
|
+
no cleanup of temporary files, does not clean up after itself
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
A sentence in the cleanup family is suppressed when, after removing
|
|
40
|
+
the matched phrase, a negation token (`no`, `not`, `never`, `without`)
|
|
41
|
+
remains — so a sentence that only denies cleanup is claimed by the
|
|
42
|
+
nocleanup family, not by both. Bare nouns such as "temporary files" or
|
|
43
|
+
"cleanup" alone are not claims.
|
|
44
|
+
|
|
45
|
+
## Evidence
|
|
46
|
+
|
|
47
|
+
Cleanup evidence in production Python source (AST only):
|
|
48
|
+
|
|
49
|
+
- attribute calls named `unlink`, `remove`, `rmtree`, or `removedirs`
|
|
50
|
+
(any receiver: `os.unlink`, `Path.unlink`, `shutil.rmtree`, ...)
|
|
51
|
+
- `TemporaryDirectory` / `TemporaryFile` calls (self-cleaning)
|
|
52
|
+
- `atexit.register` calls (teardown registration)
|
|
53
|
+
- `NamedTemporaryFile` with the `delete` argument absent or `True`
|
|
54
|
+
(self-deleting; the default is `delete=True`)
|
|
55
|
+
|
|
56
|
+
Not evidence: bare imports alone, `mkstemp` / `mkdtemp` calls
|
|
57
|
+
(creation, not cleanup), `NamedTemporaryFile(delete=False)`, `open()`
|
|
58
|
+
writes, or anything in comments and strings.
|
|
59
|
+
|
|
60
|
+
## Rules
|
|
61
|
+
|
|
62
|
+
- `documented-cleanup-claim-false` (HIGH): a cleanup claim with zero
|
|
63
|
+
production cleanup evidence; `documented-cleanup-claim-false-test-only`
|
|
64
|
+
(MEDIUM) when the evidence exists only under `tests/` or `fixtures/`.
|
|
65
|
+
- `documented-no-cleanup-claim-false` (HIGH): a no-cleanup claim with
|
|
66
|
+
production cleanup evidence; `...-test-only` (MEDIUM) for test-only
|
|
67
|
+
evidence.
|
|
68
|
+
- No production source at all yields a HIGH finding with evidence
|
|
69
|
+
`["no-source"]` for either claimed family. No claim yields PASS.
|
|
70
|
+
|
|
71
|
+
## Differentiation
|
|
72
|
+
|
|
73
|
+
- shutdown-claim-audit asks whether a process defends its teardown
|
|
74
|
+
when interrupted (signal handlers); this asks whether stray temp
|
|
75
|
+
files get deleted during a normal run, signal or no signal.
|
|
76
|
+
- atomic-claim-audit asks whether a single write lands atomically via
|
|
77
|
+
temp file plus replace; this asks whether stray temp files get
|
|
78
|
+
deleted, not how a write lands.
|
|
79
|
+
- readonly-claim-audit asks whether files are written at all; this
|
|
80
|
+
asks whether stray temp files get deleted after being written.
|
|
81
|
+
- streaming-claim-audit asks about memory behaviour; this asks about
|
|
82
|
+
temp files on disk.
|
|
83
|
+
- bandit and ruff find code issues; they never check docs promises.
|
|
84
|
+
- The `tempfile` docs describe cleanup mechanisms at runtime; they
|
|
85
|
+
never check docs against code.
|
|
86
|
+
|
|
87
|
+
## Honest limitations
|
|
88
|
+
|
|
89
|
+
- Exact phrases, not semantics: reworded promises are missed.
|
|
90
|
+
- Dead code counts as evidence, and a cleanup call anywhere in a file
|
|
91
|
+
counts even outside a `finally` block or an `atexit` handler.
|
|
92
|
+
- Receiver types are not inferred: any `.unlink()` / `.remove()` /
|
|
93
|
+
`.rmtree()` call counts, whatever the receiver is.
|
|
94
|
+
- Attribute calls always count; bare-name calls count only when
|
|
95
|
+
imported from `tempfile` / `atexit` (aliases followed).
|
|
96
|
+
- Dynamic temp paths are not tracked, and `NamedTemporaryFile` with a
|
|
97
|
+
non-literal `delete` value is not counted as cleanup.
|
|
98
|
+
- A cleanup call is not proof the right files are removed, only that
|
|
99
|
+
a cleanup mechanism exists in the source.
|
|
100
|
+
|
|
101
|
+
## License
|
|
102
|
+
|
|
103
|
+
MIT
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=61"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "cleanup-claim-audit"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Offline auditor for documented temporary-file cleanup promises in Python projects."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = "MIT"
|
|
11
|
+
requires-python = ">=3.9"
|
|
12
|
+
authors = [{name = "hao li"}]
|
|
13
|
+
|
|
14
|
+
[project.scripts]
|
|
15
|
+
cleanup-claim-audit = "cleanup_claim_audit.__main__:main"
|
|
16
|
+
|
|
17
|
+
[tool.setuptools.packages.find]
|
|
18
|
+
where = ["src"]
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
"""CLI entry point for cleanup-claim-audit."""
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import sys
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
from .audit import VERSION, audit, format_human, result_json
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class Parser(argparse.ArgumentParser):
|
|
11
|
+
def error(self, message):
|
|
12
|
+
self.print_usage(sys.stderr)
|
|
13
|
+
print("error: %s" % message, file=sys.stderr)
|
|
14
|
+
raise SystemExit(1)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def build_parser():
|
|
18
|
+
parser = Parser(prog="cleanup-claim-audit")
|
|
19
|
+
parser.add_argument("path", help="project root to audit")
|
|
20
|
+
parser.add_argument("--json", action="store_true")
|
|
21
|
+
parser.add_argument("--verbose", action="store_true")
|
|
22
|
+
parser.add_argument("--ignore", action="append", default=[],
|
|
23
|
+
help="glob matched against relative paths (repeatable)")
|
|
24
|
+
parser.add_argument("--fail-on", choices=["HIGH", "MEDIUM"], default="MEDIUM")
|
|
25
|
+
parser.add_argument("--version", action="version", version=VERSION)
|
|
26
|
+
return parser
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def main(argv=None):
|
|
30
|
+
args = build_parser().parse_args(argv)
|
|
31
|
+
root = Path(args.path)
|
|
32
|
+
if not root.is_dir():
|
|
33
|
+
print("error: not a directory: %s" % args.path, file=sys.stderr)
|
|
34
|
+
return 1
|
|
35
|
+
result = audit(root, args.ignore)
|
|
36
|
+
if args.json:
|
|
37
|
+
sys.stdout.write(result_json(result))
|
|
38
|
+
else:
|
|
39
|
+
sys.stdout.write(format_human(result))
|
|
40
|
+
if args.verbose:
|
|
41
|
+
for claim in result["verified_claims"]:
|
|
42
|
+
for phrase in claim["phrases"]:
|
|
43
|
+
print(" claim: %s" % phrase)
|
|
44
|
+
if result["status"] == "PASS":
|
|
45
|
+
return 0
|
|
46
|
+
if args.fail_on == "HIGH":
|
|
47
|
+
if all(f["severity"] != "HIGH" for f in result["findings"]):
|
|
48
|
+
return 0
|
|
49
|
+
return 2
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
if __name__ == "__main__":
|
|
53
|
+
raise SystemExit(main())
|
|
@@ -0,0 +1,278 @@
|
|
|
1
|
+
"""Core audit logic: docs temp-file cleanup claims vs Python source evidence."""
|
|
2
|
+
|
|
3
|
+
import ast
|
|
4
|
+
import fnmatch
|
|
5
|
+
import json
|
|
6
|
+
import re
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
VERSION = "0.1.0"
|
|
10
|
+
|
|
11
|
+
# Longest first so a longer phrase wins when both match one sentence;
|
|
12
|
+
# families collapse matches anyway, order only affects reported phrases.
|
|
13
|
+
CLEANUP_PHRASES = [
|
|
14
|
+
"automatically cleans up temporary files",
|
|
15
|
+
"automatic cleanup of temporary files",
|
|
16
|
+
"cleans up temporary files",
|
|
17
|
+
"clean up temporary files",
|
|
18
|
+
"cleans up its temporary files",
|
|
19
|
+
"cleans up temp files",
|
|
20
|
+
"clean up temp files",
|
|
21
|
+
"removes temporary files",
|
|
22
|
+
"remove temporary files",
|
|
23
|
+
"removes temp files",
|
|
24
|
+
"remove temp files",
|
|
25
|
+
"temporary files are cleaned up",
|
|
26
|
+
"temp files are cleaned up",
|
|
27
|
+
"temporary files are removed",
|
|
28
|
+
"temp files are removed",
|
|
29
|
+
"no temporary files left behind",
|
|
30
|
+
"leaves no temporary files",
|
|
31
|
+
"cleans up after itself",
|
|
32
|
+
"clean up after itself",
|
|
33
|
+
"temporary files cleaned up",
|
|
34
|
+
"temp files cleaned up",
|
|
35
|
+
]
|
|
36
|
+
|
|
37
|
+
NOCLEANUP_PHRASES = [
|
|
38
|
+
"does not clean up temporary files",
|
|
39
|
+
"does not clean up temp files",
|
|
40
|
+
"never cleans up temporary files",
|
|
41
|
+
"does not remove temporary files",
|
|
42
|
+
"leaves temporary files behind",
|
|
43
|
+
"temporary files are left behind",
|
|
44
|
+
"no cleanup of temporary files",
|
|
45
|
+
"does not clean up after itself",
|
|
46
|
+
]
|
|
47
|
+
|
|
48
|
+
CLEANUP = "cleanup"
|
|
49
|
+
NOCLEANUP = "nocleanup"
|
|
50
|
+
|
|
51
|
+
PHRASE_FAMILIES = (
|
|
52
|
+
[(p, CLEANUP) for p in CLEANUP_PHRASES]
|
|
53
|
+
+ [(p, NOCLEANUP) for p in NOCLEANUP_PHRASES]
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
NEGATIONS = {"no", "not", "never", "without"}
|
|
57
|
+
CLEANUP_CALL_ATTRS = {"unlink", "remove", "rmtree", "removedirs"}
|
|
58
|
+
TEMP_CTOR_NAMES = {"TemporaryDirectory", "TemporaryFile"}
|
|
59
|
+
EXCLUDE_DIRS = {"tests", "fixtures", "docs", "examples"}
|
|
60
|
+
TEST_DIRS = {"tests", "fixtures"}
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def normalize(text):
|
|
64
|
+
text = text.lower().replace("n't", " not")
|
|
65
|
+
text = text.replace("-", " ")
|
|
66
|
+
return re.sub(r"\s+", " ", text).strip()
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def strip_fenced(text):
|
|
70
|
+
lines = text.splitlines()
|
|
71
|
+
out = []
|
|
72
|
+
in_fence = False
|
|
73
|
+
for line in lines:
|
|
74
|
+
if line.strip().startswith("```"):
|
|
75
|
+
in_fence = not in_fence
|
|
76
|
+
continue
|
|
77
|
+
if not in_fence:
|
|
78
|
+
out.append(line)
|
|
79
|
+
return "\n".join(out)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def split_sentences(text):
|
|
83
|
+
parts = re.split(r"[.!?\n]+", text)
|
|
84
|
+
return [p.strip() for p in parts if p.strip()]
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _phrase_in(sentence, phrase):
|
|
88
|
+
return re.search(r"\b" + re.escape(phrase) + r"\b", sentence) is not None
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _strip_phrase(sentence, phrase):
|
|
92
|
+
return re.sub(r"\b" + re.escape(phrase) + r"\b", " ", sentence)
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def extract_claims(root, ignore):
|
|
96
|
+
"""Return {family: [phrase, ...]} found in root README + docs/**/*.md."""
|
|
97
|
+
doc_files = []
|
|
98
|
+
readme = root / "README.md"
|
|
99
|
+
if readme.is_file() and not readme.is_symlink():
|
|
100
|
+
doc_files.append(readme)
|
|
101
|
+
docs_dir = root / "docs"
|
|
102
|
+
if docs_dir.is_dir():
|
|
103
|
+
for path in sorted(docs_dir.rglob("*.md")):
|
|
104
|
+
if not path.is_symlink():
|
|
105
|
+
doc_files.append(path)
|
|
106
|
+
found = {}
|
|
107
|
+
for path in doc_files:
|
|
108
|
+
rel = path.relative_to(root).as_posix()
|
|
109
|
+
if any(fnmatch.fnmatch(rel, pat) for pat in ignore):
|
|
110
|
+
continue
|
|
111
|
+
text = strip_fenced(path.read_text(encoding="utf-8", errors="replace"))
|
|
112
|
+
for sentence in split_sentences(normalize(text)):
|
|
113
|
+
for phrase, family in PHRASE_FAMILIES:
|
|
114
|
+
if not _phrase_in(sentence, phrase):
|
|
115
|
+
continue
|
|
116
|
+
if family == CLEANUP:
|
|
117
|
+
# Remove this phrase's span first, then check the
|
|
118
|
+
# remainder for a negation token. This lets
|
|
119
|
+
# self-negating phrases ("leaves no temporary
|
|
120
|
+
# files") count while "never cleans up temporary
|
|
121
|
+
# files" is suppressed (leftover "never") in favour
|
|
122
|
+
# of its NOCLEANUP match.
|
|
123
|
+
remainder = _strip_phrase(sentence, phrase)
|
|
124
|
+
if set(remainder.split(" ")) & NEGATIONS:
|
|
125
|
+
continue
|
|
126
|
+
found.setdefault(family, [])
|
|
127
|
+
if phrase not in found[family]:
|
|
128
|
+
found[family].append(phrase)
|
|
129
|
+
return found
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def _named_temporary_file_is_evidence(node):
|
|
133
|
+
"""delete absent or True literal -> self-deleting; anything else no."""
|
|
134
|
+
for kw in node.keywords:
|
|
135
|
+
if kw.arg == "delete":
|
|
136
|
+
return isinstance(kw.value, ast.Constant) and kw.value.value is True
|
|
137
|
+
return True
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def collect_evidence(path, rel):
|
|
141
|
+
"""Return cleanup-evidence items found in one .py file (AST only)."""
|
|
142
|
+
try:
|
|
143
|
+
tree = ast.parse(path.read_text(encoding="utf-8", errors="replace"))
|
|
144
|
+
except SyntaxError:
|
|
145
|
+
return []
|
|
146
|
+
local_defs = set()
|
|
147
|
+
from_imports = {}
|
|
148
|
+
for node in ast.walk(tree):
|
|
149
|
+
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
|
|
150
|
+
local_defs.add(node.name)
|
|
151
|
+
elif isinstance(node, ast.ImportFrom):
|
|
152
|
+
if node.module in ("tempfile", "atexit"):
|
|
153
|
+
for alias in node.names:
|
|
154
|
+
from_imports[alias.asname or alias.name] = (
|
|
155
|
+
node.module, alias.name)
|
|
156
|
+
evidence = []
|
|
157
|
+
for node in ast.walk(tree):
|
|
158
|
+
if not isinstance(node, ast.Call):
|
|
159
|
+
continue
|
|
160
|
+
func = node.func
|
|
161
|
+
if isinstance(func, ast.Attribute):
|
|
162
|
+
attr = func.attr
|
|
163
|
+
if attr in CLEANUP_CALL_ATTRS:
|
|
164
|
+
evidence.append("%s:%d: %s()" % (rel, node.lineno, attr))
|
|
165
|
+
elif attr in TEMP_CTOR_NAMES:
|
|
166
|
+
evidence.append("%s:%d: %s()" % (rel, node.lineno, attr))
|
|
167
|
+
elif attr == "NamedTemporaryFile":
|
|
168
|
+
if _named_temporary_file_is_evidence(node):
|
|
169
|
+
evidence.append("%s:%d: NamedTemporaryFile()" % (rel, node.lineno))
|
|
170
|
+
elif attr == "register" and isinstance(func.value, ast.Name):
|
|
171
|
+
if func.value.id == "atexit":
|
|
172
|
+
evidence.append("%s:%d: atexit.register()" % (rel, node.lineno))
|
|
173
|
+
elif isinstance(func, ast.Name):
|
|
174
|
+
name = func.id
|
|
175
|
+
if name in local_defs:
|
|
176
|
+
continue
|
|
177
|
+
origin = from_imports.get(name)
|
|
178
|
+
if origin is None:
|
|
179
|
+
continue
|
|
180
|
+
module, orig = origin
|
|
181
|
+
if orig in TEMP_CTOR_NAMES and module == "tempfile":
|
|
182
|
+
evidence.append("%s:%d: %s()" % (rel, node.lineno, name))
|
|
183
|
+
elif orig == "NamedTemporaryFile" and module == "tempfile":
|
|
184
|
+
if _named_temporary_file_is_evidence(node):
|
|
185
|
+
evidence.append("%s:%d: NamedTemporaryFile()" % (rel, node.lineno))
|
|
186
|
+
elif orig == "register" and module == "atexit":
|
|
187
|
+
evidence.append("%s:%d: atexit.register()" % (rel, node.lineno))
|
|
188
|
+
return evidence
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def _iter_py_files(root):
|
|
192
|
+
for path in sorted(root.rglob("*.py")):
|
|
193
|
+
if path.is_symlink():
|
|
194
|
+
continue
|
|
195
|
+
yield path
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def audit(root, ignore=()):
|
|
199
|
+
claims = extract_claims(root, ignore)
|
|
200
|
+
prod_files = []
|
|
201
|
+
prod_evidence = []
|
|
202
|
+
test_evidence = []
|
|
203
|
+
for path in _iter_py_files(root):
|
|
204
|
+
rel = path.relative_to(root).as_posix()
|
|
205
|
+
if any(fnmatch.fnmatch(rel, pat) for pat in ignore):
|
|
206
|
+
continue
|
|
207
|
+
parts = set(path.relative_to(root).parts[:-1])
|
|
208
|
+
name = path.name
|
|
209
|
+
if parts & TEST_DIRS:
|
|
210
|
+
test_evidence.extend(collect_evidence(path, rel))
|
|
211
|
+
elif parts & EXCLUDE_DIRS:
|
|
212
|
+
continue
|
|
213
|
+
elif name.startswith("test_") or name.endswith("_test.py"):
|
|
214
|
+
test_evidence.extend(collect_evidence(path, rel))
|
|
215
|
+
else:
|
|
216
|
+
prod_files.append(path)
|
|
217
|
+
prod_evidence.extend(collect_evidence(path, rel))
|
|
218
|
+
findings = []
|
|
219
|
+
verified = []
|
|
220
|
+
|
|
221
|
+
def handle(family, false_rule, testonly_rule):
|
|
222
|
+
if family not in claims:
|
|
223
|
+
return
|
|
224
|
+
if prod_evidence:
|
|
225
|
+
verified.append({"family": family, "phrases": claims[family]})
|
|
226
|
+
elif test_evidence:
|
|
227
|
+
findings.append({"rule": testonly_rule, "severity": "MEDIUM",
|
|
228
|
+
"family": family, "evidence": sorted(test_evidence)})
|
|
229
|
+
elif not prod_files:
|
|
230
|
+
findings.append({"rule": false_rule, "severity": "HIGH",
|
|
231
|
+
"family": family, "evidence": ["no-source"]})
|
|
232
|
+
else:
|
|
233
|
+
findings.append({"rule": false_rule, "severity": "HIGH",
|
|
234
|
+
"family": family, "evidence": []})
|
|
235
|
+
|
|
236
|
+
# Reverse rule: a no-cleanup claim is falsified by production
|
|
237
|
+
# cleanup evidence, so handle it with the roles swapped.
|
|
238
|
+
if CLEANUP in claims:
|
|
239
|
+
handle(CLEANUP, "documented-cleanup-claim-false",
|
|
240
|
+
"documented-cleanup-claim-false-test-only")
|
|
241
|
+
if NOCLEANUP in claims:
|
|
242
|
+
if prod_evidence:
|
|
243
|
+
findings.append({"rule": "documented-no-cleanup-claim-false",
|
|
244
|
+
"severity": "HIGH", "family": NOCLEANUP,
|
|
245
|
+
"evidence": sorted(prod_evidence)})
|
|
246
|
+
elif test_evidence:
|
|
247
|
+
findings.append({"rule": "documented-no-cleanup-claim-false-test-only",
|
|
248
|
+
"severity": "MEDIUM", "family": NOCLEANUP,
|
|
249
|
+
"evidence": sorted(test_evidence)})
|
|
250
|
+
elif not prod_files:
|
|
251
|
+
findings.append({"rule": "documented-no-cleanup-claim-false",
|
|
252
|
+
"severity": "HIGH", "family": NOCLEANUP,
|
|
253
|
+
"evidence": ["no-source"]})
|
|
254
|
+
else:
|
|
255
|
+
verified.append({"family": NOCLEANUP, "phrases": claims[NOCLEANUP]})
|
|
256
|
+
return {
|
|
257
|
+
"status": "PASS" if not findings else "FAIL",
|
|
258
|
+
"findings": findings,
|
|
259
|
+
"verified_claims": verified,
|
|
260
|
+
"claims": claims,
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
def result_json(result):
|
|
265
|
+
return json.dumps(result, indent=2, sort_keys=True) + "\n"
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
def format_human(result):
|
|
269
|
+
lines = ["status: %s" % result["status"]]
|
|
270
|
+
for finding in result["findings"]:
|
|
271
|
+
lines.append("%s %s (%s)" % (finding["severity"], finding["rule"],
|
|
272
|
+
finding["family"]))
|
|
273
|
+
for item in finding["evidence"]:
|
|
274
|
+
lines.append(" evidence: %s" % item)
|
|
275
|
+
for claim in result["verified_claims"]:
|
|
276
|
+
lines.append("verified: %s (%s)" % (claim["family"],
|
|
277
|
+
", ".join(claim["phrases"])))
|
|
278
|
+
return "\n".join(lines) + "\n"
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: cleanup-claim-audit
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Offline auditor for documented temporary-file cleanup promises in Python projects.
|
|
5
|
+
Author: hao li
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Requires-Python: >=3.9
|
|
8
|
+
Description-Content-Type: text/markdown
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Dynamic: license-file
|
|
11
|
+
|
|
12
|
+
# cleanup-claim-audit
|
|
13
|
+
|
|
14
|
+
Offline auditor for documented temporary-file cleanup promises in Python
|
|
15
|
+
projects. It checks whether a project's docs promise temp-file tidiness
|
|
16
|
+
(or promise the opposite) and whether its Python source backs that
|
|
17
|
+
promise up. It executes nothing, uses no network, and
|
|
18
|
+
never reads `.env` file contents.
|
|
19
|
+
|
|
20
|
+
## Usage
|
|
21
|
+
|
|
22
|
+
```sh
|
|
23
|
+
cleanup-claim-audit <project-root>
|
|
24
|
+
cleanup-claim-audit <project-root> --json
|
|
25
|
+
cleanup-claim-audit <project-root> --fail-on HIGH
|
|
26
|
+
cleanup-claim-audit <project-root> --ignore 'docs/draft-*'
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
Exit codes: `0` PASS, `2` FAIL, `1` usage error. `--fail-on HIGH` makes
|
|
30
|
+
MEDIUM-only results exit `0`. `--json` prints
|
|
31
|
+
`{status, findings, verified_claims, claims}`.
|
|
32
|
+
|
|
33
|
+
## Claims checked
|
|
34
|
+
|
|
35
|
+
Claims come from prose in the root `README.md` plus `docs/**/*.md`.
|
|
36
|
+
Fenced code blocks are excluded; inline code counts as prose. Claim
|
|
37
|
+
phrases are exact phrases (see below, shown in a fenced block so this
|
|
38
|
+
very README does not itself make a claim the tool would audit):
|
|
39
|
+
|
|
40
|
+
```
|
|
41
|
+
cleanup family: cleans up temporary files, cleans up temp files,
|
|
42
|
+
removes temporary files, temporary files are cleaned up,
|
|
43
|
+
no temporary files left behind, leaves no temporary files,
|
|
44
|
+
cleans up after itself, automatically cleans up temporary files
|
|
45
|
+
nocleanup family: does not clean up temporary files,
|
|
46
|
+
never cleans up temporary files, leaves temporary files behind,
|
|
47
|
+
no cleanup of temporary files, does not clean up after itself
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
A sentence in the cleanup family is suppressed when, after removing
|
|
51
|
+
the matched phrase, a negation token (`no`, `not`, `never`, `without`)
|
|
52
|
+
remains — so a sentence that only denies cleanup is claimed by the
|
|
53
|
+
nocleanup family, not by both. Bare nouns such as "temporary files" or
|
|
54
|
+
"cleanup" alone are not claims.
|
|
55
|
+
|
|
56
|
+
## Evidence
|
|
57
|
+
|
|
58
|
+
Cleanup evidence in production Python source (AST only):
|
|
59
|
+
|
|
60
|
+
- attribute calls named `unlink`, `remove`, `rmtree`, or `removedirs`
|
|
61
|
+
(any receiver: `os.unlink`, `Path.unlink`, `shutil.rmtree`, ...)
|
|
62
|
+
- `TemporaryDirectory` / `TemporaryFile` calls (self-cleaning)
|
|
63
|
+
- `atexit.register` calls (teardown registration)
|
|
64
|
+
- `NamedTemporaryFile` with the `delete` argument absent or `True`
|
|
65
|
+
(self-deleting; the default is `delete=True`)
|
|
66
|
+
|
|
67
|
+
Not evidence: bare imports alone, `mkstemp` / `mkdtemp` calls
|
|
68
|
+
(creation, not cleanup), `NamedTemporaryFile(delete=False)`, `open()`
|
|
69
|
+
writes, or anything in comments and strings.
|
|
70
|
+
|
|
71
|
+
## Rules
|
|
72
|
+
|
|
73
|
+
- `documented-cleanup-claim-false` (HIGH): a cleanup claim with zero
|
|
74
|
+
production cleanup evidence; `documented-cleanup-claim-false-test-only`
|
|
75
|
+
(MEDIUM) when the evidence exists only under `tests/` or `fixtures/`.
|
|
76
|
+
- `documented-no-cleanup-claim-false` (HIGH): a no-cleanup claim with
|
|
77
|
+
production cleanup evidence; `...-test-only` (MEDIUM) for test-only
|
|
78
|
+
evidence.
|
|
79
|
+
- No production source at all yields a HIGH finding with evidence
|
|
80
|
+
`["no-source"]` for either claimed family. No claim yields PASS.
|
|
81
|
+
|
|
82
|
+
## Differentiation
|
|
83
|
+
|
|
84
|
+
- shutdown-claim-audit asks whether a process defends its teardown
|
|
85
|
+
when interrupted (signal handlers); this asks whether stray temp
|
|
86
|
+
files get deleted during a normal run, signal or no signal.
|
|
87
|
+
- atomic-claim-audit asks whether a single write lands atomically via
|
|
88
|
+
temp file plus replace; this asks whether stray temp files get
|
|
89
|
+
deleted, not how a write lands.
|
|
90
|
+
- readonly-claim-audit asks whether files are written at all; this
|
|
91
|
+
asks whether stray temp files get deleted after being written.
|
|
92
|
+
- streaming-claim-audit asks about memory behaviour; this asks about
|
|
93
|
+
temp files on disk.
|
|
94
|
+
- bandit and ruff find code issues; they never check docs promises.
|
|
95
|
+
- The `tempfile` docs describe cleanup mechanisms at runtime; they
|
|
96
|
+
never check docs against code.
|
|
97
|
+
|
|
98
|
+
## Honest limitations
|
|
99
|
+
|
|
100
|
+
- Exact phrases, not semantics: reworded promises are missed.
|
|
101
|
+
- Dead code counts as evidence, and a cleanup call anywhere in a file
|
|
102
|
+
counts even outside a `finally` block or an `atexit` handler.
|
|
103
|
+
- Receiver types are not inferred: any `.unlink()` / `.remove()` /
|
|
104
|
+
`.rmtree()` call counts, whatever the receiver is.
|
|
105
|
+
- Attribute calls always count; bare-name calls count only when
|
|
106
|
+
imported from `tempfile` / `atexit` (aliases followed).
|
|
107
|
+
- Dynamic temp paths are not tracked, and `NamedTemporaryFile` with a
|
|
108
|
+
non-literal `delete` value is not counted as cleanup.
|
|
109
|
+
- A cleanup call is not proof the right files are removed, only that
|
|
110
|
+
a cleanup mechanism exists in the source.
|
|
111
|
+
|
|
112
|
+
## License
|
|
113
|
+
|
|
114
|
+
MIT
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
MANIFEST.in
|
|
3
|
+
README.md
|
|
4
|
+
pyproject.toml
|
|
5
|
+
src/cleanup_claim_audit/__init__.py
|
|
6
|
+
src/cleanup_claim_audit/__main__.py
|
|
7
|
+
src/cleanup_claim_audit/audit.py
|
|
8
|
+
src/cleanup_claim_audit.egg-info/PKG-INFO
|
|
9
|
+
src/cleanup_claim_audit.egg-info/SOURCES.txt
|
|
10
|
+
src/cleanup_claim_audit.egg-info/dependency_links.txt
|
|
11
|
+
src/cleanup_claim_audit.egg-info/entry_points.txt
|
|
12
|
+
src/cleanup_claim_audit.egg-info/top_level.txt
|
|
13
|
+
tests/test_audit.py
|
|
14
|
+
tests/fixtures/bad/cleanup/.env
|
|
15
|
+
tests/fixtures/bad/cleanup/README.md
|
|
16
|
+
tests/fixtures/bad/cleanup/main.py
|
|
17
|
+
tests/fixtures/bad/nocleanup/README.md
|
|
18
|
+
tests/fixtures/bad/nocleanup/main.py
|
|
19
|
+
tests/fixtures/good/cleanup/README.md
|
|
20
|
+
tests/fixtures/good/cleanup/main.py
|
|
21
|
+
tests/fixtures/good/nocleanup/README.md
|
|
22
|
+
tests/fixtures/good/nocleanup/main.py
|
|
23
|
+
tests/fixtures/testonly/cleanup/README.md
|
|
24
|
+
tests/fixtures/testonly/cleanup/main.py
|
|
25
|
+
tests/fixtures/testonly/cleanup/tests/test_work.py
|
|
26
|
+
tests/fixtures/testonly/nocleanup/README.md
|
|
27
|
+
tests/fixtures/testonly/nocleanup/main.py
|
|
28
|
+
tests/fixtures/testonly/nocleanup/tests/test_work.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
cleanup_claim_audit
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
CLEANUP_CLAIM_AUDIT_MARKER_DO_NOT_LEAK=1
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import shutil
|
|
2
|
+
import tempfile
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
def run():
|
|
6
|
+
with tempfile.TemporaryDirectory() as work:
|
|
7
|
+
scratch = work + "/out.txt"
|
|
8
|
+
with open(scratch, "w", encoding="utf-8") as handle:
|
|
9
|
+
handle.write("done")
|
|
10
|
+
shutil.rmtree(work, ignore_errors=True)
|
|
11
|
+
return True
|
|
@@ -0,0 +1,397 @@
|
|
|
1
|
+
"""Tests for cleanup-claim-audit (run with PYTHONPATH=src)."""
|
|
2
|
+
|
|
3
|
+
import io
|
|
4
|
+
import json
|
|
5
|
+
import os
|
|
6
|
+
import sys
|
|
7
|
+
import tempfile
|
|
8
|
+
import unittest
|
|
9
|
+
from contextlib import redirect_stdout
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
|
|
12
|
+
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
|
|
13
|
+
|
|
14
|
+
from cleanup_claim_audit.audit import ( # noqa: E402
|
|
15
|
+
CLEANUP_PHRASES, NOCLEANUP_PHRASES, audit, collect_evidence,
|
|
16
|
+
extract_claims)
|
|
17
|
+
from cleanup_claim_audit.__main__ import main # noqa: E402
|
|
18
|
+
|
|
19
|
+
FIXTURES = Path(__file__).resolve().parent / "fixtures"
|
|
20
|
+
MARKER = "CLEANUP_CLAIM_AUDIT_MARKER_DO_NOT_LEAK"
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def make_project(readme=None, files=None, docs=None):
|
|
24
|
+
tmp = tempfile.mkdtemp()
|
|
25
|
+
root = Path(tmp)
|
|
26
|
+
if readme is not None:
|
|
27
|
+
(root / "README.md").write_text(readme, encoding="utf-8")
|
|
28
|
+
for rel, text in (files or {}).items():
|
|
29
|
+
path = root / rel
|
|
30
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
31
|
+
path.write_text(text, encoding="utf-8")
|
|
32
|
+
for rel, text in (docs or {}).items():
|
|
33
|
+
path = root / "docs" / rel
|
|
34
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
35
|
+
path.write_text(text, encoding="utf-8")
|
|
36
|
+
return root
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def run_cli(root, *args):
|
|
40
|
+
buf = io.StringIO()
|
|
41
|
+
with redirect_stdout(buf):
|
|
42
|
+
code = main([str(root)] + list(args))
|
|
43
|
+
return code, buf.getvalue()
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
class ClaimTests(unittest.TestCase):
|
|
47
|
+
def claims_for(self, readme, **kw):
|
|
48
|
+
return extract_claims(make_project(readme=readme, **kw), ())
|
|
49
|
+
|
|
50
|
+
def test_all_cleanup_phrases(self):
|
|
51
|
+
for phrase in CLEANUP_PHRASES:
|
|
52
|
+
with self.subTest(phrase=phrase):
|
|
53
|
+
found = self.claims_for("Tool. It %s every run." % phrase)
|
|
54
|
+
self.assertIn("cleanup", found)
|
|
55
|
+
self.assertIn(phrase, found["cleanup"])
|
|
56
|
+
|
|
57
|
+
def test_all_nocleanup_phrases(self):
|
|
58
|
+
for phrase in NOCLEANUP_PHRASES:
|
|
59
|
+
with self.subTest(phrase=phrase):
|
|
60
|
+
found = self.claims_for("Tool. It %s, by design." % phrase)
|
|
61
|
+
self.assertIn("nocleanup", found)
|
|
62
|
+
self.assertIn(phrase, found["nocleanup"])
|
|
63
|
+
|
|
64
|
+
def test_never_cleans_single_family(self):
|
|
65
|
+
found = self.claims_for("It never cleans up temporary files.")
|
|
66
|
+
self.assertEqual(set(found), {"nocleanup"})
|
|
67
|
+
|
|
68
|
+
def test_does_not_clean_single_family(self):
|
|
69
|
+
found = self.claims_for("It does not clean up temporary files.")
|
|
70
|
+
self.assertEqual(set(found), {"nocleanup"})
|
|
71
|
+
|
|
72
|
+
def test_does_not_remove_temp_single_family(self):
|
|
73
|
+
found = self.claims_for("It does not remove temporary files.")
|
|
74
|
+
self.assertEqual(set(found), {"nocleanup"})
|
|
75
|
+
|
|
76
|
+
def test_leaves_no_counts_cleanup(self):
|
|
77
|
+
found = self.claims_for("It leaves no temporary files.")
|
|
78
|
+
self.assertEqual(set(found), {"cleanup"})
|
|
79
|
+
|
|
80
|
+
def test_no_left_behind_counts_cleanup(self):
|
|
81
|
+
found = self.claims_for("No temporary files left behind.")
|
|
82
|
+
self.assertEqual(set(found), {"cleanup"})
|
|
83
|
+
|
|
84
|
+
def test_never_removes_suppressed_no_nocleanup_phrase(self):
|
|
85
|
+
# "never removes temp files" is not a NOCLEANUP phrase; the
|
|
86
|
+
# CLEANUP match is suppressed by the leftover "never".
|
|
87
|
+
found = self.claims_for("It never removes temp files.")
|
|
88
|
+
self.assertEqual(found, {})
|
|
89
|
+
|
|
90
|
+
def test_does_not_remove_temp_suppressed(self):
|
|
91
|
+
found = self.claims_for("It does not remove temp files.")
|
|
92
|
+
self.assertEqual(found, {})
|
|
93
|
+
|
|
94
|
+
def test_fenced_code_excluded(self):
|
|
95
|
+
found = self.claims_for(
|
|
96
|
+
"Plain intro.\n```\nIt cleans up temporary files.\n```\n")
|
|
97
|
+
self.assertEqual(found, {})
|
|
98
|
+
|
|
99
|
+
def test_inline_code_counts(self):
|
|
100
|
+
found = self.claims_for(
|
|
101
|
+
"The tool `cleans up temporary files` after each run.")
|
|
102
|
+
self.assertIn("cleanup", found)
|
|
103
|
+
|
|
104
|
+
def test_bare_nouns_no_claim(self):
|
|
105
|
+
found = self.claims_for(
|
|
106
|
+
"Temporary files and cleanup are discussed here at length.")
|
|
107
|
+
self.assertEqual(found, {})
|
|
108
|
+
|
|
109
|
+
def test_bare_temp_files_no_claim(self):
|
|
110
|
+
found = self.claims_for("It writes temp files to disk.")
|
|
111
|
+
self.assertEqual(found, {})
|
|
112
|
+
|
|
113
|
+
def test_docs_subdir_counts(self):
|
|
114
|
+
found = self.claims_for(
|
|
115
|
+
"Intro.", docs={"guide.md": "It cleans up temporary files."})
|
|
116
|
+
self.assertIn("cleanup", found)
|
|
117
|
+
|
|
118
|
+
def test_docs_fenced_excluded(self):
|
|
119
|
+
found = self.claims_for(
|
|
120
|
+
"Intro.",
|
|
121
|
+
docs={"guide.md": "```\nIt cleans up temporary files.\n```\n"})
|
|
122
|
+
self.assertEqual(found, {})
|
|
123
|
+
|
|
124
|
+
def test_ignore_glob(self):
|
|
125
|
+
root = make_project(
|
|
126
|
+
readme="Intro.",
|
|
127
|
+
docs={"guide.md": "It cleans up temporary files."})
|
|
128
|
+
self.assertEqual(extract_claims(root, ("docs/*",)), {})
|
|
129
|
+
|
|
130
|
+
def test_case_and_hyphen_normalised(self):
|
|
131
|
+
found = self.claims_for(
|
|
132
|
+
"It Cleans-Up Temporary Files after every run.")
|
|
133
|
+
self.assertIn("cleanup", found)
|
|
134
|
+
|
|
135
|
+
def test_symlink_doc_skipped(self):
|
|
136
|
+
root = make_project()
|
|
137
|
+
target = root / "target.md"
|
|
138
|
+
target.write_text("It cleans up temporary files.", encoding="utf-8")
|
|
139
|
+
os.symlink(target, root / "README.md")
|
|
140
|
+
# README.md is a symlink; root README must be skipped.
|
|
141
|
+
self.assertEqual(extract_claims(root, ()), {})
|
|
142
|
+
|
|
143
|
+
def test_no_readme_no_claim(self):
|
|
144
|
+
self.assertEqual(extract_claims(make_project(), ()), {})
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
class EvidenceTests(unittest.TestCase):
|
|
148
|
+
def evidence_for(self, source):
|
|
149
|
+
tmp = tempfile.mkdtemp()
|
|
150
|
+
path = Path(tmp) / "main.py"
|
|
151
|
+
path.write_text(source, encoding="utf-8")
|
|
152
|
+
return collect_evidence(path, "main.py")
|
|
153
|
+
|
|
154
|
+
def test_os_unlink(self):
|
|
155
|
+
self.assertTrue(self.evidence_for("import os\nos.unlink(p)\n"))
|
|
156
|
+
|
|
157
|
+
def test_path_unlink(self):
|
|
158
|
+
self.assertTrue(self.evidence_for("p.unlink()\n"))
|
|
159
|
+
|
|
160
|
+
def test_os_remove(self):
|
|
161
|
+
self.assertTrue(self.evidence_for("import os\nos.remove(p)\n"))
|
|
162
|
+
|
|
163
|
+
def test_shutil_rmtree(self):
|
|
164
|
+
self.assertTrue(self.evidence_for(
|
|
165
|
+
"import shutil\nshutil.rmtree(d)\n"))
|
|
166
|
+
|
|
167
|
+
def test_os_removedirs(self):
|
|
168
|
+
self.assertTrue(self.evidence_for("import os\nos.removedirs(d)\n"))
|
|
169
|
+
|
|
170
|
+
def test_tempdir_attr(self):
|
|
171
|
+
self.assertTrue(self.evidence_for(
|
|
172
|
+
"import tempfile\ntempfile.TemporaryDirectory()\n"))
|
|
173
|
+
|
|
174
|
+
def test_tempfile_attr(self):
|
|
175
|
+
self.assertTrue(self.evidence_for(
|
|
176
|
+
"import tempfile\ntempfile.TemporaryFile()\n"))
|
|
177
|
+
|
|
178
|
+
def test_atexit_register_attr(self):
|
|
179
|
+
self.assertTrue(self.evidence_for(
|
|
180
|
+
"import atexit\natexit.register(f)\n"))
|
|
181
|
+
|
|
182
|
+
def test_bare_tempdir_from_import(self):
|
|
183
|
+
self.assertTrue(self.evidence_for(
|
|
184
|
+
"from tempfile import TemporaryDirectory\nTemporaryDirectory()\n"))
|
|
185
|
+
|
|
186
|
+
def test_bare_tempfile_from_import(self):
|
|
187
|
+
self.assertTrue(self.evidence_for(
|
|
188
|
+
"from tempfile import TemporaryFile\nTemporaryFile()\n"))
|
|
189
|
+
|
|
190
|
+
def test_bare_register_from_import(self):
|
|
191
|
+
self.assertTrue(self.evidence_for(
|
|
192
|
+
"from atexit import register\nregister(f)\n"))
|
|
193
|
+
|
|
194
|
+
def test_aliased_tempdir_from_import(self):
|
|
195
|
+
self.assertTrue(self.evidence_for(
|
|
196
|
+
"from tempfile import TemporaryDirectory as TD\nTD()\n"))
|
|
197
|
+
|
|
198
|
+
def test_named_temp_default(self):
|
|
199
|
+
self.assertTrue(self.evidence_for(
|
|
200
|
+
"import tempfile\ntempfile.NamedTemporaryFile()\n"))
|
|
201
|
+
|
|
202
|
+
def test_named_temp_delete_true(self):
|
|
203
|
+
self.assertTrue(self.evidence_for(
|
|
204
|
+
"import tempfile\ntempfile.NamedTemporaryFile(delete=True)\n"))
|
|
205
|
+
|
|
206
|
+
def test_named_temp_delete_false_not_evidence(self):
|
|
207
|
+
self.assertEqual(self.evidence_for(
|
|
208
|
+
"import tempfile\ntempfile.NamedTemporaryFile(delete=False)\n"), [])
|
|
209
|
+
|
|
210
|
+
def test_named_temp_delete_variable_not_evidence(self):
|
|
211
|
+
self.assertEqual(self.evidence_for(
|
|
212
|
+
"import tempfile\ntempfile.NamedTemporaryFile(delete=flag)\n"), [])
|
|
213
|
+
|
|
214
|
+
def test_mkstemp_not_evidence(self):
|
|
215
|
+
self.assertEqual(self.evidence_for(
|
|
216
|
+
"import tempfile\ntempfile.mkstemp()\n"), [])
|
|
217
|
+
|
|
218
|
+
def test_mkdtemp_not_evidence(self):
|
|
219
|
+
self.assertEqual(self.evidence_for(
|
|
220
|
+
"import tempfile\ntempfile.mkdtemp()\n"), [])
|
|
221
|
+
|
|
222
|
+
def test_bare_import_not_evidence(self):
|
|
223
|
+
self.assertEqual(self.evidence_for(
|
|
224
|
+
"import tempfile\nimport os\nimport shutil\nimport atexit\n"), [])
|
|
225
|
+
|
|
226
|
+
def test_bare_name_without_import_not_evidence(self):
|
|
227
|
+
self.assertEqual(self.evidence_for("TemporaryDirectory()\n"), [])
|
|
228
|
+
|
|
229
|
+
def test_shadowed_bare_name_not_evidence(self):
|
|
230
|
+
self.assertEqual(self.evidence_for(
|
|
231
|
+
"from tempfile import TemporaryDirectory\n"
|
|
232
|
+
"def TemporaryDirectory():\n pass\nTemporaryDirectory()\n"), [])
|
|
233
|
+
|
|
234
|
+
def test_comment_not_evidence(self):
|
|
235
|
+
self.assertEqual(self.evidence_for("# os.unlink(p)\n"), [])
|
|
236
|
+
|
|
237
|
+
def test_string_not_evidence(self):
|
|
238
|
+
self.assertEqual(self.evidence_for('x = "os.unlink(p)"\n'), [])
|
|
239
|
+
|
|
240
|
+
def test_syntax_error_no_evidence(self):
|
|
241
|
+
self.assertEqual(self.evidence_for("def broken(:\n"), [])
|
|
242
|
+
|
|
243
|
+
def test_open_write_not_evidence(self):
|
|
244
|
+
self.assertEqual(self.evidence_for(
|
|
245
|
+
'open("/tmp/x", "w").write("hi")\n'), [])
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
class AuditTests(unittest.TestCase):
|
|
249
|
+
def test_good_cleanup_pass(self):
|
|
250
|
+
result = audit(FIXTURES / "good" / "cleanup")
|
|
251
|
+
self.assertEqual(result["status"], "PASS")
|
|
252
|
+
self.assertEqual(len(result["verified_claims"]), 1)
|
|
253
|
+
self.assertEqual(result["verified_claims"][0]["family"], "cleanup")
|
|
254
|
+
|
|
255
|
+
def test_good_nocleanup_pass(self):
|
|
256
|
+
result = audit(FIXTURES / "good" / "nocleanup")
|
|
257
|
+
self.assertEqual(result["status"], "PASS")
|
|
258
|
+
self.assertEqual(result["verified_claims"][0]["family"], "nocleanup")
|
|
259
|
+
|
|
260
|
+
def test_bad_cleanup_exact_high(self):
|
|
261
|
+
result = audit(FIXTURES / "bad" / "cleanup")
|
|
262
|
+
self.assertEqual(result["status"], "FAIL")
|
|
263
|
+
self.assertEqual(len(result["findings"]), 1)
|
|
264
|
+
self.assertEqual(result["findings"][0]["rule"],
|
|
265
|
+
"documented-cleanup-claim-false")
|
|
266
|
+
self.assertEqual(result["findings"][0]["severity"], "HIGH")
|
|
267
|
+
|
|
268
|
+
def test_bad_nocleanup_exact_high(self):
|
|
269
|
+
result = audit(FIXTURES / "bad" / "nocleanup")
|
|
270
|
+
self.assertEqual(result["status"], "FAIL")
|
|
271
|
+
self.assertEqual(len(result["findings"]), 1)
|
|
272
|
+
self.assertEqual(result["findings"][0]["rule"],
|
|
273
|
+
"documented-no-cleanup-claim-false")
|
|
274
|
+
self.assertEqual(result["findings"][0]["severity"], "HIGH")
|
|
275
|
+
|
|
276
|
+
def test_testonly_cleanup_medium(self):
|
|
277
|
+
result = audit(FIXTURES / "testonly" / "cleanup")
|
|
278
|
+
self.assertEqual(result["status"], "FAIL")
|
|
279
|
+
self.assertEqual(result["findings"][0]["rule"],
|
|
280
|
+
"documented-cleanup-claim-false-test-only")
|
|
281
|
+
self.assertEqual(result["findings"][0]["severity"], "MEDIUM")
|
|
282
|
+
|
|
283
|
+
def test_testonly_nocleanup_medium(self):
|
|
284
|
+
result = audit(FIXTURES / "testonly" / "nocleanup")
|
|
285
|
+
self.assertEqual(result["status"], "FAIL")
|
|
286
|
+
self.assertEqual(result["findings"][0]["rule"],
|
|
287
|
+
"documented-no-cleanup-claim-false-test-only")
|
|
288
|
+
self.assertEqual(result["findings"][0]["severity"], "MEDIUM")
|
|
289
|
+
|
|
290
|
+
def test_no_source_cleanup_high(self):
|
|
291
|
+
root = make_project(
|
|
292
|
+
readme="It cleans up temporary files after every run.")
|
|
293
|
+
result = audit(root)
|
|
294
|
+
self.assertEqual(result["findings"][0]["severity"], "HIGH")
|
|
295
|
+
self.assertEqual(result["findings"][0]["evidence"], ["no-source"])
|
|
296
|
+
|
|
297
|
+
def test_no_source_nocleanup_high(self):
|
|
298
|
+
root = make_project(
|
|
299
|
+
readme="It does not clean up temporary files.")
|
|
300
|
+
result = audit(root)
|
|
301
|
+
self.assertEqual(result["findings"][0]["rule"],
|
|
302
|
+
"documented-no-cleanup-claim-false")
|
|
303
|
+
self.assertEqual(result["findings"][0]["evidence"], ["no-source"])
|
|
304
|
+
|
|
305
|
+
def test_no_claim_pass(self):
|
|
306
|
+
root = make_project(readme="A plain tool.", files={
|
|
307
|
+
"main.py": "import os\nos.unlink(p)\n"})
|
|
308
|
+
self.assertEqual(audit(root)["status"], "PASS")
|
|
309
|
+
|
|
310
|
+
def test_mixed_claims_cleanup_fails_nocleanup_verified(self):
|
|
311
|
+
root = make_project(
|
|
312
|
+
readme="It cleans up temporary files. It does not clean up "
|
|
313
|
+
"temporary files.",
|
|
314
|
+
files={"main.py": "import tempfile\ntempfile.mkstemp()\n"})
|
|
315
|
+
result = audit(root)
|
|
316
|
+
self.assertEqual(len(result["findings"]), 1)
|
|
317
|
+
self.assertEqual(result["findings"][0]["rule"],
|
|
318
|
+
"documented-cleanup-claim-false")
|
|
319
|
+
self.assertEqual([c["family"] for c in result["verified_claims"]],
|
|
320
|
+
["nocleanup"])
|
|
321
|
+
|
|
322
|
+
def test_test_file_at_root_counts_as_test(self):
|
|
323
|
+
root = make_project(
|
|
324
|
+
readme="It cleans up temporary files.",
|
|
325
|
+
files={"test_main.py": "import shutil\nshutil.rmtree(d)\n"})
|
|
326
|
+
result = audit(root)
|
|
327
|
+
self.assertEqual(result["findings"][0]["severity"], "MEDIUM")
|
|
328
|
+
|
|
329
|
+
def test_examples_dir_excluded(self):
|
|
330
|
+
root = make_project(
|
|
331
|
+
readme="It cleans up temporary files.",
|
|
332
|
+
files={"examples/demo.py": "import shutil\nshutil.rmtree(d)\n"})
|
|
333
|
+
result = audit(root)
|
|
334
|
+
self.assertEqual(result["findings"][0]["evidence"], ["no-source"])
|
|
335
|
+
|
|
336
|
+
def test_marker_never_leaks(self):
|
|
337
|
+
code, out = run_cli(FIXTURES / "bad" / "cleanup", "--json")
|
|
338
|
+
self.assertEqual(code, 2)
|
|
339
|
+
self.assertNotIn(MARKER, out)
|
|
340
|
+
|
|
341
|
+
def test_self_audit_pass(self):
|
|
342
|
+
root = Path(__file__).resolve().parents[1]
|
|
343
|
+
result = audit(root)
|
|
344
|
+
self.assertEqual(result["status"], "PASS")
|
|
345
|
+
self.assertEqual(result["claims"], {})
|
|
346
|
+
|
|
347
|
+
|
|
348
|
+
class CliTests(unittest.TestCase):
|
|
349
|
+
def test_good_exit_0(self):
|
|
350
|
+
code, out = run_cli(FIXTURES / "good" / "cleanup")
|
|
351
|
+
self.assertEqual(code, 0)
|
|
352
|
+
self.assertIn("status: PASS", out)
|
|
353
|
+
|
|
354
|
+
def test_bad_exit_2(self):
|
|
355
|
+
code, out = run_cli(FIXTURES / "bad" / "cleanup")
|
|
356
|
+
self.assertEqual(code, 2)
|
|
357
|
+
self.assertIn("documented-cleanup-claim-false", out)
|
|
358
|
+
|
|
359
|
+
def test_bad_path_exit_1(self):
|
|
360
|
+
code, _ = run_cli(Path("/nonexistent-cleanup-audit-root"))
|
|
361
|
+
self.assertEqual(code, 1)
|
|
362
|
+
|
|
363
|
+
def test_json_valid(self):
|
|
364
|
+
code, out = run_cli(FIXTURES / "bad" / "nocleanup", "--json")
|
|
365
|
+
self.assertEqual(code, 2)
|
|
366
|
+
parsed = json.loads(out)
|
|
367
|
+
self.assertEqual(parsed["status"], "FAIL")
|
|
368
|
+
self.assertEqual(parsed["findings"][0]["family"], "nocleanup")
|
|
369
|
+
|
|
370
|
+
def test_fail_on_high_testonly_exit_0(self):
|
|
371
|
+
code, _ = run_cli(FIXTURES / "testonly" / "cleanup",
|
|
372
|
+
"--fail-on", "HIGH")
|
|
373
|
+
self.assertEqual(code, 0)
|
|
374
|
+
|
|
375
|
+
def test_fail_on_high_bad_exit_2(self):
|
|
376
|
+
code, _ = run_cli(FIXTURES / "bad" / "cleanup",
|
|
377
|
+
"--fail-on", "HIGH")
|
|
378
|
+
self.assertEqual(code, 2)
|
|
379
|
+
|
|
380
|
+
def test_verbose_prints_claims(self):
|
|
381
|
+
code, out = run_cli(FIXTURES / "good" / "cleanup", "--verbose")
|
|
382
|
+
self.assertEqual(code, 0)
|
|
383
|
+
self.assertIn("claim:", out)
|
|
384
|
+
|
|
385
|
+
def test_version(self):
|
|
386
|
+
with self.assertRaises(SystemExit) as caught:
|
|
387
|
+
main(["--version"])
|
|
388
|
+
self.assertEqual(caught.exception.code, 0)
|
|
389
|
+
|
|
390
|
+
def test_bad_flag_exit_1(self):
|
|
391
|
+
with self.assertRaises(SystemExit) as caught:
|
|
392
|
+
main([str(FIXTURES / "good" / "cleanup"), "--bogus"])
|
|
393
|
+
self.assertEqual(caught.exception.code, 1)
|
|
394
|
+
|
|
395
|
+
|
|
396
|
+
if __name__ == "__main__":
|
|
397
|
+
unittest.main()
|