assert-pytest-test-can-fail 20260908004717__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- assert_pytest_test_can_fail/__init__.py +0 -0
- assert_pytest_test_can_fail/__main__.py +3 -0
- assert_pytest_test_can_fail/cli.py +274 -0
- assert_pytest_test_can_fail/py.typed +0 -0
- assert_pytest_test_can_fail/scanner.py +116 -0
- assert_pytest_test_can_fail-20260908004717.dist-info/METADATA +186 -0
- assert_pytest_test_can_fail-20260908004717.dist-info/RECORD +11 -0
- assert_pytest_test_can_fail-20260908004717.dist-info/WHEEL +5 -0
- assert_pytest_test_can_fail-20260908004717.dist-info/entry_points.txt +2 -0
- assert_pytest_test_can_fail-20260908004717.dist-info/licenses/LICENSE.txt +190 -0
- assert_pytest_test_can_fail-20260908004717.dist-info/top_level.txt +1 -0
|
File without changes
|
|
@@ -0,0 +1,274 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import fnmatch
|
|
5
|
+
import glob
|
|
6
|
+
import os
|
|
7
|
+
import sys
|
|
8
|
+
from dataclasses import dataclass
|
|
9
|
+
from typing import TYPE_CHECKING
|
|
10
|
+
|
|
11
|
+
from .scanner import Finding, is_test_file, scan_file
|
|
12
|
+
|
|
13
|
+
if TYPE_CHECKING:
|
|
14
|
+
from collections.abc import Sequence
|
|
15
|
+
|
|
16
|
+
EXIT_SUCCESS = 0
|
|
17
|
+
EXIT_FINDINGS = 1
|
|
18
|
+
EXIT_ERROR = 2
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@dataclass
|
|
22
|
+
class ScanResult:
|
|
23
|
+
findings: list[Finding]
|
|
24
|
+
files_scanned: int
|
|
25
|
+
had_error: bool
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def create_parser() -> argparse.ArgumentParser:
|
|
29
|
+
parser = argparse.ArgumentParser(
|
|
30
|
+
prog="assert-pytest-test-can-fail",
|
|
31
|
+
description="Assert that every pytest test is able to fail.",
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
parser.add_argument(
|
|
35
|
+
"files",
|
|
36
|
+
nargs="+",
|
|
37
|
+
metavar="FILE",
|
|
38
|
+
help=(
|
|
39
|
+
"One or more file paths, directory paths, or glob patterns to scan. "
|
|
40
|
+
"Directories are scanned recursively for test files (test_*.py, *_test.py)."
|
|
41
|
+
),
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
parser.add_argument(
|
|
45
|
+
"--exclude",
|
|
46
|
+
metavar="PATTERNS",
|
|
47
|
+
help="Comma-separated glob patterns to exclude files.",
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
output_group = parser.add_mutually_exclusive_group()
|
|
51
|
+
output_group.add_argument(
|
|
52
|
+
"--quiet",
|
|
53
|
+
action="store_true",
|
|
54
|
+
help="Suppress all output. Exit code indicates success (0) or findings (1).",
|
|
55
|
+
)
|
|
56
|
+
output_group.add_argument(
|
|
57
|
+
"--count",
|
|
58
|
+
action="store_true",
|
|
59
|
+
help="Output only the count of findings.",
|
|
60
|
+
)
|
|
61
|
+
output_group.add_argument(
|
|
62
|
+
"--verbose",
|
|
63
|
+
action="store_true",
|
|
64
|
+
help="Show detailed processing information.",
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
behavior_group = parser.add_mutually_exclusive_group()
|
|
68
|
+
behavior_group.add_argument(
|
|
69
|
+
"--fail-fast",
|
|
70
|
+
action="store_true",
|
|
71
|
+
help="Exit immediately after finding the first issue.",
|
|
72
|
+
)
|
|
73
|
+
behavior_group.add_argument(
|
|
74
|
+
"--warn-only",
|
|
75
|
+
action="store_true",
|
|
76
|
+
help="Always exit with code 0, even if findings exist.",
|
|
77
|
+
)
|
|
78
|
+
|
|
79
|
+
return parser
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def parse_patterns(patterns_str: str | None) -> list[str]:
|
|
83
|
+
if not patterns_str:
|
|
84
|
+
return []
|
|
85
|
+
return [p.strip() for p in patterns_str.split(",") if p.strip()]
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _is_glob_pattern(path: str) -> bool:
|
|
89
|
+
return any(c in path for c in ("*", "?", "["))
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def _expand_glob(pattern: str) -> tuple[list[str], bool]:
|
|
93
|
+
matched = glob.glob(pattern, recursive=True, include_hidden=True)
|
|
94
|
+
files = []
|
|
95
|
+
for path in matched:
|
|
96
|
+
if os.path.isfile(path):
|
|
97
|
+
files.append(path)
|
|
98
|
+
elif os.path.isdir(path):
|
|
99
|
+
files.extend(_expand_directory(path))
|
|
100
|
+
return (files, bool(matched))
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def _expand_directory(directory: str) -> list[str]:
|
|
104
|
+
files = []
|
|
105
|
+
for root, dirs, filenames in os.walk(directory):
|
|
106
|
+
dirs[:] = [d for d in dirs if not d.startswith(".")]
|
|
107
|
+
|
|
108
|
+
for filename in filenames:
|
|
109
|
+
if filename.endswith(".py"):
|
|
110
|
+
filepath = os.path.join(root, filename)
|
|
111
|
+
if is_test_file(filepath):
|
|
112
|
+
files.append(filepath)
|
|
113
|
+
return files
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def _iter_files(paths: Sequence[str]) -> tuple[list[str], list[str]]:
|
|
117
|
+
files: list[str] = []
|
|
118
|
+
missing: list[str] = []
|
|
119
|
+
|
|
120
|
+
for path in paths:
|
|
121
|
+
if _is_glob_pattern(path):
|
|
122
|
+
matched_files, found = _expand_glob(path)
|
|
123
|
+
if found:
|
|
124
|
+
files.extend(matched_files)
|
|
125
|
+
else:
|
|
126
|
+
missing.append(path)
|
|
127
|
+
elif os.path.isfile(path):
|
|
128
|
+
files.append(path)
|
|
129
|
+
elif os.path.isdir(path):
|
|
130
|
+
files.extend(_expand_directory(path))
|
|
131
|
+
else:
|
|
132
|
+
missing.append(path)
|
|
133
|
+
|
|
134
|
+
seen: set[str] = set()
|
|
135
|
+
unique_files: list[str] = []
|
|
136
|
+
for f in files:
|
|
137
|
+
normalized = os.path.normpath(f)
|
|
138
|
+
if normalized not in seen:
|
|
139
|
+
seen.add(normalized)
|
|
140
|
+
unique_files.append(f)
|
|
141
|
+
|
|
142
|
+
return (unique_files, missing)
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def _should_skip_file(path: str, exclude_patterns: list[str]) -> bool:
|
|
146
|
+
if not exclude_patterns:
|
|
147
|
+
return False
|
|
148
|
+
|
|
149
|
+
for pattern in exclude_patterns:
|
|
150
|
+
if fnmatch.fnmatch(path, pattern):
|
|
151
|
+
return True
|
|
152
|
+
if fnmatch.fnmatch(os.path.basename(path), pattern):
|
|
153
|
+
return True
|
|
154
|
+
return False
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def process_files(
|
|
158
|
+
files: list[str],
|
|
159
|
+
exclude_patterns: list[str],
|
|
160
|
+
verbose: bool = False,
|
|
161
|
+
fail_fast: bool = False,
|
|
162
|
+
) -> ScanResult:
|
|
163
|
+
findings: list[Finding] = []
|
|
164
|
+
files_scanned = 0
|
|
165
|
+
had_error = False
|
|
166
|
+
|
|
167
|
+
for path in sorted(files):
|
|
168
|
+
if not path.endswith(".py"):
|
|
169
|
+
continue
|
|
170
|
+
|
|
171
|
+
if not is_test_file(path):
|
|
172
|
+
continue
|
|
173
|
+
|
|
174
|
+
if _should_skip_file(path, exclude_patterns):
|
|
175
|
+
if verbose:
|
|
176
|
+
print(f"Skipping (excluded): {path}")
|
|
177
|
+
continue
|
|
178
|
+
|
|
179
|
+
if verbose:
|
|
180
|
+
print(f"Scanning: {path}")
|
|
181
|
+
|
|
182
|
+
try:
|
|
183
|
+
with open(path, encoding="utf-8") as f:
|
|
184
|
+
content = f.read()
|
|
185
|
+
except OSError as e:
|
|
186
|
+
print(f"Error reading {path}: {e}", file=sys.stderr)
|
|
187
|
+
had_error = True
|
|
188
|
+
continue
|
|
189
|
+
|
|
190
|
+
try:
|
|
191
|
+
file_findings = scan_file(path, content)
|
|
192
|
+
except SyntaxError as e:
|
|
193
|
+
print(f"Syntax error in {path}: {e}", file=sys.stderr)
|
|
194
|
+
had_error = True
|
|
195
|
+
continue
|
|
196
|
+
|
|
197
|
+
files_scanned += 1
|
|
198
|
+
findings.extend(file_findings)
|
|
199
|
+
|
|
200
|
+
if file_findings and verbose:
|
|
201
|
+
for finding in file_findings:
|
|
202
|
+
print(f" Found: {finding}")
|
|
203
|
+
|
|
204
|
+
if findings and fail_fast:
|
|
205
|
+
break
|
|
206
|
+
|
|
207
|
+
return ScanResult(
|
|
208
|
+
findings=findings,
|
|
209
|
+
files_scanned=files_scanned,
|
|
210
|
+
had_error=had_error,
|
|
211
|
+
)
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def output_findings(
|
|
215
|
+
findings: list[Finding],
|
|
216
|
+
count_mode: bool = False,
|
|
217
|
+
) -> None:
|
|
218
|
+
if count_mode:
|
|
219
|
+
print(len(findings))
|
|
220
|
+
else:
|
|
221
|
+
for finding in findings:
|
|
222
|
+
print(finding)
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def determine_exit_code(result: ScanResult, warn_only: bool = False) -> int:
|
|
226
|
+
if warn_only:
|
|
227
|
+
return EXIT_SUCCESS
|
|
228
|
+
if result.findings:
|
|
229
|
+
return EXIT_FINDINGS
|
|
230
|
+
if result.had_error:
|
|
231
|
+
return EXIT_ERROR
|
|
232
|
+
return EXIT_SUCCESS
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def main(argv: Sequence[str] | None = None) -> None:
|
|
236
|
+
parser = create_parser()
|
|
237
|
+
args = parser.parse_args(argv)
|
|
238
|
+
|
|
239
|
+
exclude_patterns = parse_patterns(args.exclude)
|
|
240
|
+
|
|
241
|
+
files, missing = _iter_files(args.files)
|
|
242
|
+
|
|
243
|
+
for path in missing:
|
|
244
|
+
print(f"Error: Path not found: {path}", file=sys.stderr)
|
|
245
|
+
|
|
246
|
+
if not files and missing:
|
|
247
|
+
sys.exit(EXIT_ERROR)
|
|
248
|
+
|
|
249
|
+
if args.verbose:
|
|
250
|
+
print(f"Scanning {len(files)} file(s) for tests that cannot fail...")
|
|
251
|
+
if exclude_patterns:
|
|
252
|
+
print(f"Excluding patterns: {', '.join(exclude_patterns)}")
|
|
253
|
+
print()
|
|
254
|
+
|
|
255
|
+
result = process_files(
|
|
256
|
+
files=files,
|
|
257
|
+
exclude_patterns=exclude_patterns,
|
|
258
|
+
verbose=args.verbose,
|
|
259
|
+
fail_fast=args.fail_fast,
|
|
260
|
+
)
|
|
261
|
+
|
|
262
|
+
if missing:
|
|
263
|
+
result.had_error = True
|
|
264
|
+
|
|
265
|
+
if args.verbose:
|
|
266
|
+
print()
|
|
267
|
+
print(f"Files scanned: {result.files_scanned}")
|
|
268
|
+
print(f"Findings: {len(result.findings)}")
|
|
269
|
+
if result.had_error:
|
|
270
|
+
print("Errors occurred during scanning.")
|
|
271
|
+
elif not args.quiet:
|
|
272
|
+
output_findings(result.findings, args.count)
|
|
273
|
+
|
|
274
|
+
sys.exit(determine_exit_code(result, args.warn_only))
|
|
File without changes
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import ast
|
|
4
|
+
import os
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
from typing import TYPE_CHECKING
|
|
7
|
+
|
|
8
|
+
if TYPE_CHECKING:
|
|
9
|
+
from collections.abc import Iterator
|
|
10
|
+
|
|
11
|
+
MESSAGE = "every path to the assert sets the flag, so the test cannot fail"
|
|
12
|
+
|
|
13
|
+
FUNCTIONS = (ast.FunctionDef, ast.AsyncFunctionDef)
|
|
14
|
+
SCOPES = (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef, ast.Lambda)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@dataclass(frozen=True)
|
|
18
|
+
class Finding:
|
|
19
|
+
path: str
|
|
20
|
+
line_number: int
|
|
21
|
+
function_name: str
|
|
22
|
+
|
|
23
|
+
def __str__(self) -> str:
|
|
24
|
+
return f"{self.path}:{self.line_number}:{self.function_name}: {MESSAGE}"
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def is_test_function(name: str) -> bool:
|
|
28
|
+
return name.startswith("test")
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def is_test_file(path: str) -> bool:
|
|
32
|
+
basename = os.path.basename(path)
|
|
33
|
+
return basename.startswith("test_") or basename.endswith("_test.py")
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _sets_the_flag(statement: ast.stmt, flag: str) -> bool:
|
|
37
|
+
if not isinstance(statement, ast.Assign) or len(statement.targets) != 1:
|
|
38
|
+
return False
|
|
39
|
+
target = statement.targets[0]
|
|
40
|
+
if not isinstance(target, ast.Name) or target.id != flag:
|
|
41
|
+
return False
|
|
42
|
+
value = statement.value
|
|
43
|
+
return isinstance(value, ast.Constant) and value.value is True
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _closes(body: list[ast.stmt], flag: str) -> bool:
|
|
47
|
+
if not body:
|
|
48
|
+
return False
|
|
49
|
+
last = body[-1]
|
|
50
|
+
if isinstance(last, ast.Raise) or _sets_the_flag(last, flag):
|
|
51
|
+
return True
|
|
52
|
+
if isinstance(last, ast.If):
|
|
53
|
+
return _closes(last.body, flag) and _closes(last.orelse, flag)
|
|
54
|
+
return False
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def cannot_fail(node: ast.Try, flag: str) -> bool:
|
|
58
|
+
if not node.handlers:
|
|
59
|
+
return False
|
|
60
|
+
if not _closes(node.orelse or node.body, flag):
|
|
61
|
+
return False
|
|
62
|
+
return all(_closes(handler.body, flag) for handler in node.handlers)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _descend(node: ast.AST) -> Iterator[ast.AST]:
|
|
66
|
+
for child in ast.iter_child_nodes(node):
|
|
67
|
+
yield child
|
|
68
|
+
if not isinstance(child, SCOPES):
|
|
69
|
+
yield from _descend(child)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _tries(node: ast.AST) -> list[ast.Try]:
|
|
73
|
+
return [child for child in _descend(node) if isinstance(child, ast.Try)]
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _flag_assertions(node: ast.AST) -> list[tuple[int, str]]:
|
|
77
|
+
assertions: list[tuple[int, str]] = []
|
|
78
|
+
for child in _descend(node):
|
|
79
|
+
if not isinstance(child, ast.Assert):
|
|
80
|
+
continue
|
|
81
|
+
flag = child.test
|
|
82
|
+
if isinstance(flag, ast.Name):
|
|
83
|
+
assertions.append((child.lineno, flag.id))
|
|
84
|
+
return assertions
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _unfailable_assertion(node: ast.FunctionDef | ast.AsyncFunctionDef) -> int | None:
|
|
88
|
+
for statement in _tries(node):
|
|
89
|
+
end = statement.end_lineno or statement.lineno
|
|
90
|
+
for line_number, flag in _flag_assertions(node):
|
|
91
|
+
if line_number > end and cannot_fail(statement, flag):
|
|
92
|
+
return line_number
|
|
93
|
+
return None
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
class UnfailableFinder(ast.NodeVisitor):
|
|
97
|
+
def __init__(self, path: str) -> None:
|
|
98
|
+
self.path = path
|
|
99
|
+
self.findings: list[Finding] = []
|
|
100
|
+
|
|
101
|
+
def generic_visit(self, node: ast.AST) -> None:
|
|
102
|
+
if isinstance(node, FUNCTIONS) and is_test_function(node.name):
|
|
103
|
+
self._check_function(node)
|
|
104
|
+
super().generic_visit(node)
|
|
105
|
+
|
|
106
|
+
def _check_function(self, node: ast.FunctionDef | ast.AsyncFunctionDef) -> None:
|
|
107
|
+
line_number = _unfailable_assertion(node)
|
|
108
|
+
if line_number is not None:
|
|
109
|
+
self.findings.append(Finding(self.path, line_number, node.name))
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def scan_file(path: str, content: str) -> list[Finding]:
|
|
113
|
+
tree = ast.parse(content, filename=path)
|
|
114
|
+
finder = UnfailableFinder(path)
|
|
115
|
+
finder.visit(tree)
|
|
116
|
+
return finder.findings
|
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: assert-pytest-test-can-fail
|
|
3
|
+
Version: 20260908004717
|
|
4
|
+
Summary: CLI tool to assert that every pytest test is able to fail
|
|
5
|
+
Author-email: 10U Labs <dev@10ulabs.com>
|
|
6
|
+
License-Expression: Apache-2.0
|
|
7
|
+
Project-URL: Homepage, https://github.com/10U-Labs/assert-pytest-test-can-fail
|
|
8
|
+
Project-URL: Repository, https://github.com/10U-Labs/assert-pytest-test-can-fail
|
|
9
|
+
Classifier: Development Status :: 4 - Beta
|
|
10
|
+
Classifier: Environment :: Console
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: Operating System :: OS Independent
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
18
|
+
Classifier: Topic :: Software Development :: Quality Assurance
|
|
19
|
+
Classifier: Topic :: Software Development :: Testing
|
|
20
|
+
Requires-Python: >=3.10
|
|
21
|
+
Description-Content-Type: text/markdown
|
|
22
|
+
License-File: LICENSE.txt
|
|
23
|
+
Dynamic: license-file
|
|
24
|
+
|
|
25
|
+
# assert-pytest-test-can-fail
|
|
26
|
+
|
|
27
|
+
Assert that every pytest test is able to fail.
|
|
28
|
+
|
|
29
|
+
## Why Must a Test Be Able to Fail?
|
|
30
|
+
|
|
31
|
+
A test earns its place by being able to fail. A test whose assertion is
|
|
32
|
+
satisfied on every path it can reach runs, costs what it costs, and reports
|
|
33
|
+
green for the whole life of the thing it claims to watch. Coverage counts it.
|
|
34
|
+
The suite counts it. The only way anybody finds out is by reading it.
|
|
35
|
+
|
|
36
|
+
One shape produces this reliably. A test sets a flag to `False`, asks a remote
|
|
37
|
+
service about a resource, sets the flag to `True` when the call succeeds, and
|
|
38
|
+
sets the same flag to `True` again in the handler for the error that means the
|
|
39
|
+
resource is absent. Every other error is re-raised. Then it asserts the flag.
|
|
40
|
+
Every path that reaches the assertion has set it to `True`, so the assertion is
|
|
41
|
+
decoration, and a genuinely missing resource is reported as present.
|
|
42
|
+
|
|
43
|
+
Mutation testing answers this question in general and answers it better, by
|
|
44
|
+
mutating the code under test and seeing whether a test dies. It cannot answer
|
|
45
|
+
it for a test with no code under test — one that asserts about a deployed
|
|
46
|
+
resource through a client library — and it costs a suite run per mutant. This
|
|
47
|
+
tool asks a narrower question that a parse can settle.
|
|
48
|
+
|
|
49
|
+
## Installation
|
|
50
|
+
|
|
51
|
+
```bash
|
|
52
|
+
pip install assert-pytest-test-can-fail
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
## Usage
|
|
56
|
+
|
|
57
|
+
### Command Line
|
|
58
|
+
|
|
59
|
+
```bash
|
|
60
|
+
# Scan specific files
|
|
61
|
+
assert-pytest-test-can-fail test_example.py
|
|
62
|
+
|
|
63
|
+
# Scan directories recursively
|
|
64
|
+
assert-pytest-test-can-fail test/
|
|
65
|
+
|
|
66
|
+
# Use glob patterns
|
|
67
|
+
assert-pytest-test-can-fail "test/**/test_*.py"
|
|
68
|
+
|
|
69
|
+
# Exclude patterns
|
|
70
|
+
assert-pytest-test-can-fail test/ --exclude "**/conftest.py"
|
|
71
|
+
|
|
72
|
+
# Verbose output
|
|
73
|
+
assert-pytest-test-can-fail test/ --verbose
|
|
74
|
+
|
|
75
|
+
# Fail fast (exit on first finding)
|
|
76
|
+
assert-pytest-test-can-fail test/ --fail-fast
|
|
77
|
+
|
|
78
|
+
# Warn only (always exit 0)
|
|
79
|
+
assert-pytest-test-can-fail test/ --warn-only
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
### GitHub Actions
|
|
83
|
+
|
|
84
|
+
```yaml
|
|
85
|
+
- uses: 10U-Labs/assert-pytest-test-can-fail@latest
|
|
86
|
+
with:
|
|
87
|
+
files: "test/"
|
|
88
|
+
verbose: "true"
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
### As a Python Module
|
|
92
|
+
|
|
93
|
+
```bash
|
|
94
|
+
python -m assert_pytest_test_can_fail test/
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
## Output Format
|
|
98
|
+
|
|
99
|
+
Default output shows one finding per line:
|
|
100
|
+
|
|
101
|
+
```text
|
|
102
|
+
path/to/test_file.py:23:test_bucket_exists: every path to the assert sets the flag, so the test cannot fail
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
Format: `file_path:line_number:function_name`, followed by the reason. The line
|
|
106
|
+
number is the `assert` statement.
|
|
107
|
+
|
|
108
|
+
## Exit Codes
|
|
109
|
+
|
|
110
|
+
- `0`: No findings (or `--warn-only` specified)
|
|
111
|
+
- `1`: Findings detected
|
|
112
|
+
- `2`: Error (missing files, syntax errors, etc.)
|
|
113
|
+
|
|
114
|
+
## What Is Reported
|
|
115
|
+
|
|
116
|
+
A function whose name begins with `test` is reported when all of these are
|
|
117
|
+
true:
|
|
118
|
+
|
|
119
|
+
- it contains a `try` statement with at least one `except` handler
|
|
120
|
+
- the statement that ends the `try` body — or the `else` clause, when there is
|
|
121
|
+
one — assigns `True` to a bare name
|
|
122
|
+
- every handler on that `try` is closed: it ends in `raise`, or it ends
|
|
123
|
+
assigning `True` to that same name, or it ends in an `if`/`else` whose two
|
|
124
|
+
branches each do one of those
|
|
125
|
+
- an `assert` on that bare name follows the whole `try` statement
|
|
126
|
+
|
|
127
|
+
The last one matters. An assertion sitting inside the `try`, or above it, is
|
|
128
|
+
asking a different question, and is left alone.
|
|
129
|
+
|
|
130
|
+
Each reported function is reported once, at the first assertion that qualifies.
|
|
131
|
+
|
|
132
|
+
## What Is Never Reported
|
|
133
|
+
|
|
134
|
+
The shape alone is not the defect, and a rule that says otherwise is a rule
|
|
135
|
+
nobody keeps. A permission check written this way is correct: it asks whether
|
|
136
|
+
the credentials may inspect a resource, a 404 answers yes, and any error code
|
|
137
|
+
the handler does not name leaves the flag `False`, so the assertion fails.
|
|
138
|
+
|
|
139
|
+
```python
|
|
140
|
+
def test_can_describe_bucket():
|
|
141
|
+
has_permission = False
|
|
142
|
+
try:
|
|
143
|
+
client.head_bucket(Bucket=name)
|
|
144
|
+
has_permission = True
|
|
145
|
+
except ClientError as error:
|
|
146
|
+
code = error.response["Error"]["Code"]
|
|
147
|
+
if code == "403":
|
|
148
|
+
pytest.fail("No permission to inspect the bucket")
|
|
149
|
+
if code == "404":
|
|
150
|
+
has_permission = True
|
|
151
|
+
assert has_permission
|
|
152
|
+
```
|
|
153
|
+
|
|
154
|
+
That handler ends in an `if` with no `else`, so it can complete with the flag
|
|
155
|
+
still `False`. It is not reported. Neither is a handler that ends in `pass`,
|
|
156
|
+
nor a `pytest.fail` that leaves other codes falling through — only `raise` and
|
|
157
|
+
an assignment on every branch close a handler.
|
|
158
|
+
|
|
159
|
+
Also never reported:
|
|
160
|
+
|
|
161
|
+
- a `try` with no `except` handler at all, such as `try`/`finally`
|
|
162
|
+
- a success path that computes the flag rather than asserting it, such as
|
|
163
|
+
`exists = client.bucket_exists(name)`
|
|
164
|
+
- a chained assignment, `exists = asked = True`, or an attribute target,
|
|
165
|
+
`record.exists = True`
|
|
166
|
+
- an assertion on anything but a bare name, such as `assert count == 1`
|
|
167
|
+
- a `try` inside a nested function, a lambda or a class the test defines: the
|
|
168
|
+
tool does not descend into a scope the test opens
|
|
169
|
+
- a function whose name does not begin with `test`
|
|
170
|
+
|
|
171
|
+
Only files pytest would collect are scanned: `test_*.py` and `*_test.py`.
|
|
172
|
+
|
|
173
|
+
## Options
|
|
174
|
+
|
|
175
|
+
| Option | Description |
|
|
176
|
+
| --------------------- | ------------------------------------------ |
|
|
177
|
+
| `--exclude PATTERNS` | Glob patterns to exclude (comma-separated) |
|
|
178
|
+
| `--quiet` | Suppress all output (exit code only) |
|
|
179
|
+
| `--count` | Output only the number of findings |
|
|
180
|
+
| `--verbose` | Show detailed processing information |
|
|
181
|
+
| `--fail-fast` | Exit after first finding |
|
|
182
|
+
| `--warn-only` | Always exit 0, even with findings |
|
|
183
|
+
|
|
184
|
+
## License
|
|
185
|
+
|
|
186
|
+
Apache 2.0 - See [LICENSE.txt](LICENSE.txt)
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
assert_pytest_test_can_fail/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
2
|
+
assert_pytest_test_can_fail/__main__.py,sha256=bYt9eEaoRQWdejEHFD8REx9jxVEdZptECFsV7F49Ink,30
|
|
3
|
+
assert_pytest_test_can_fail/cli.py,sha256=l7KE2fYO-X2ma-Ao-iVt3aPW6h7PNoVZ0qRsxdPRtS8,7205
|
|
4
|
+
assert_pytest_test_can_fail/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
5
|
+
assert_pytest_test_can_fail/scanner.py,sha256=XLkKFFKYKciQIyn0RuCkOMvKHIQ8LpZBAhOKUGsZ1rs,3583
|
|
6
|
+
assert_pytest_test_can_fail-20260908004717.dist-info/licenses/LICENSE.txt,sha256=7C7FkuzwSQFE_PL6st5x2YyMN1aiyjZ9ti1s5X45no8,10761
|
|
7
|
+
assert_pytest_test_can_fail-20260908004717.dist-info/METADATA,sha256=u9Z607tzovyAgI69jBJBUH_myK6SMKbAItycwBCL6vw,6575
|
|
8
|
+
assert_pytest_test_can_fail-20260908004717.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
|
|
9
|
+
assert_pytest_test_can_fail-20260908004717.dist-info/entry_points.txt,sha256=c5fmCjgD9HXmn3_nzHMcFb_zSC-V4wyRHdTxFLOC0RU,85
|
|
10
|
+
assert_pytest_test_can_fail-20260908004717.dist-info/top_level.txt,sha256=thkvO2fzN5_9lXjKvlMnWVMu8aCXOO4vUI5mOdEE-mc,28
|
|
11
|
+
assert_pytest_test_can_fail-20260908004717.dist-info/RECORD,,
|
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
Apache License
|
|
2
|
+
Version 2.0, January 2004
|
|
3
|
+
http://www.apache.org/licenses/
|
|
4
|
+
|
|
5
|
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
|
6
|
+
|
|
7
|
+
1. Definitions.
|
|
8
|
+
|
|
9
|
+
"License" shall mean the terms and conditions for use, reproduction,
|
|
10
|
+
and distribution as defined by Sections 1 through 9 of this document.
|
|
11
|
+
|
|
12
|
+
"Licensor" shall mean the copyright owner or entity authorized by
|
|
13
|
+
the copyright owner that is granting the License.
|
|
14
|
+
|
|
15
|
+
"Legal Entity" shall mean the union of the acting entity and all
|
|
16
|
+
other entities that control, are controlled by, or are under common
|
|
17
|
+
control with that entity. For the purposes of this definition,
|
|
18
|
+
"control" means (i) the power, direct or indirect, to cause the
|
|
19
|
+
direction or management of such entity, whether by contract or
|
|
20
|
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
|
21
|
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
|
22
|
+
|
|
23
|
+
"You" (or "Your") shall mean an individual or Legal Entity
|
|
24
|
+
exercising permissions granted by this License.
|
|
25
|
+
|
|
26
|
+
"Source" form shall mean the preferred form for making modifications,
|
|
27
|
+
including but not limited to software source code, documentation
|
|
28
|
+
source, and configuration files.
|
|
29
|
+
|
|
30
|
+
"Object" form shall mean any form resulting from mechanical
|
|
31
|
+
transformation or translation of a Source form, including but
|
|
32
|
+
not limited to compiled object code, generated documentation,
|
|
33
|
+
and conversions to other media types.
|
|
34
|
+
|
|
35
|
+
"Work" shall mean the work of authorship, whether in Source or
|
|
36
|
+
Object form, made available under the License, as indicated by a
|
|
37
|
+
copyright notice that is included in or attached to the work
|
|
38
|
+
(an example is provided in the Appendix below).
|
|
39
|
+
|
|
40
|
+
"Derivative Works" shall mean any work, whether in Source or Object
|
|
41
|
+
form, that is based on (or derived from) the Work and for which the
|
|
42
|
+
editorial revisions, annotations, elaborations, or other modifications
|
|
43
|
+
represent, as a whole, an original work of authorship. For the purposes
|
|
44
|
+
of this License, Derivative Works shall not include works that remain
|
|
45
|
+
separable from, or merely link (or bind by name) to the interfaces of,
|
|
46
|
+
the Work and Derivative Works thereof.
|
|
47
|
+
|
|
48
|
+
"Contribution" shall mean any work of authorship, including
|
|
49
|
+
the original version of the Work and any modifications or additions
|
|
50
|
+
to that Work or Derivative Works thereof, that is intentionally
|
|
51
|
+
submitted to the Licensor for inclusion in the Work by the copyright owner
|
|
52
|
+
or by an individual or Legal Entity authorized to submit on behalf of
|
|
53
|
+
the copyright owner. For the purposes of this definition, "submitted"
|
|
54
|
+
means any form of electronic, verbal, or written communication sent
|
|
55
|
+
to the Licensor or its representatives, including but not limited to
|
|
56
|
+
communication on electronic mailing lists, source code control systems,
|
|
57
|
+
and issue tracking systems that are managed by, or on behalf of, the
|
|
58
|
+
Licensor for the purpose of discussing and improving the Work, but
|
|
59
|
+
excluding communication that is conspicuously marked or otherwise
|
|
60
|
+
designated in writing by the copyright owner as "Not a Contribution."
|
|
61
|
+
|
|
62
|
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
|
63
|
+
on behalf of whom a Contribution has been received by Licensor and
|
|
64
|
+
subsequently incorporated within the Work.
|
|
65
|
+
|
|
66
|
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
|
67
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
68
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
69
|
+
copyright license to reproduce, prepare Derivative Works of,
|
|
70
|
+
publicly display, publicly perform, sublicense, and distribute the
|
|
71
|
+
Work and such Derivative Works in Source or Object form.
|
|
72
|
+
|
|
73
|
+
3. Grant of Patent License. Subject to the terms and conditions of
|
|
74
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
75
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
76
|
+
(except as stated in this section) patent license to make, have made,
|
|
77
|
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
|
78
|
+
where such license applies only to those patent claims licensable
|
|
79
|
+
by such Contributor that are necessarily infringed by their
|
|
80
|
+
Contribution(s) alone or by combination of their Contribution(s)
|
|
81
|
+
with the Work to which such Contribution(s) was submitted. If You
|
|
82
|
+
institute patent litigation against any entity (including a
|
|
83
|
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
|
84
|
+
or a Contribution incorporated within the Work constitutes direct
|
|
85
|
+
or contributory patent infringement, then any patent licenses
|
|
86
|
+
granted to You under this License for that Work shall terminate
|
|
87
|
+
as of the date such litigation is filed.
|
|
88
|
+
|
|
89
|
+
4. Redistribution. You may reproduce and distribute copies of the
|
|
90
|
+
Work or Derivative Works thereof in any medium, with or without
|
|
91
|
+
modifications, and in Source or Object form, provided that You
|
|
92
|
+
meet the following conditions:
|
|
93
|
+
|
|
94
|
+
(a) You must give any other recipients of the Work or
|
|
95
|
+
Derivative Works a copy of this License; and
|
|
96
|
+
|
|
97
|
+
(b) You must cause any modified files to carry prominent notices
|
|
98
|
+
stating that You changed the files; and
|
|
99
|
+
|
|
100
|
+
(c) You must retain, in the Source form of any Derivative Works
|
|
101
|
+
that You distribute, all copyright, patent, trademark, and
|
|
102
|
+
attribution notices from the Source form of the Work,
|
|
103
|
+
excluding those notices that do not pertain to any part of
|
|
104
|
+
the Derivative Works; and
|
|
105
|
+
|
|
106
|
+
(d) If the Work includes a "NOTICE" text file as part of its
|
|
107
|
+
distribution, then any Derivative Works that You distribute must
|
|
108
|
+
include a readable copy of the attribution notices contained
|
|
109
|
+
within such NOTICE file, excluding those notices that do not
|
|
110
|
+
pertain to any part of the Derivative Works, in at least one
|
|
111
|
+
of the following places: within a NOTICE text file distributed
|
|
112
|
+
as part of the Derivative Works; within the Source form or
|
|
113
|
+
documentation, if provided along with the Derivative Works; or,
|
|
114
|
+
within a display generated by the Derivative Works, if and
|
|
115
|
+
wherever such third-party notices normally appear. The contents
|
|
116
|
+
of the NOTICE file are for informational purposes only and
|
|
117
|
+
do not modify the License. You may add Your own attribution
|
|
118
|
+
notices within Derivative Works that You distribute, alongside
|
|
119
|
+
or as an addendum to the NOTICE text from the Work, provided
|
|
120
|
+
that such additional attribution notices cannot be construed
|
|
121
|
+
as modifying the License.
|
|
122
|
+
|
|
123
|
+
You may add Your own copyright statement to Your modifications and
|
|
124
|
+
may provide additional or different license terms and conditions
|
|
125
|
+
for use, reproduction, or distribution of Your modifications, or
|
|
126
|
+
for any such Derivative Works as a whole, provided Your use,
|
|
127
|
+
reproduction, and distribution of the Work otherwise complies with
|
|
128
|
+
the conditions stated in this License.
|
|
129
|
+
|
|
130
|
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
|
131
|
+
any Contribution intentionally submitted for inclusion in the Work
|
|
132
|
+
by You to the Licensor shall be under the terms and conditions of
|
|
133
|
+
this License, without any additional terms or conditions.
|
|
134
|
+
Notwithstanding the above, nothing herein shall supersede or modify
|
|
135
|
+
the terms of any separate license agreement you may have executed
|
|
136
|
+
with Licensor regarding such Contributions.
|
|
137
|
+
|
|
138
|
+
6. Trademarks. This License does not grant permission to use the trade
|
|
139
|
+
names, trademarks, service marks, or product names of the Licensor,
|
|
140
|
+
except as required for reasonable and customary use in describing the
|
|
141
|
+
origin of the Work and reproducing the content of the NOTICE file.
|
|
142
|
+
|
|
143
|
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
|
144
|
+
agreed to in writing, Licensor provides the Work (and each
|
|
145
|
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
|
146
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
147
|
+
implied, including, without limitation, any warranties or conditions
|
|
148
|
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
|
149
|
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
|
150
|
+
appropriateness of using or redistributing the Work and assume any
|
|
151
|
+
risks associated with Your exercise of permissions under this License.
|
|
152
|
+
|
|
153
|
+
8. Limitation of Liability. In no event and under no legal theory,
|
|
154
|
+
whether in tort (including negligence), contract, or otherwise,
|
|
155
|
+
unless required by applicable law (such as deliberate and grossly
|
|
156
|
+
negligent acts) or agreed to in writing, shall any Contributor be
|
|
157
|
+
liable to You for damages, including any direct, indirect, special,
|
|
158
|
+
incidental, or consequential damages of any character arising as a
|
|
159
|
+
result of this License or out of the use or inability to use the
|
|
160
|
+
Work (including but not limited to damages for loss of goodwill,
|
|
161
|
+
work stoppage, computer failure or malfunction, or any and all
|
|
162
|
+
other commercial damages or losses), even if such Contributor
|
|
163
|
+
has been advised of the possibility of such damages.
|
|
164
|
+
|
|
165
|
+
9. Accepting Warranty or Additional Liability. While redistributing
|
|
166
|
+
the Work or Derivative Works thereof, You may choose to offer,
|
|
167
|
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
|
168
|
+
or other liability obligations and/or rights consistent with this
|
|
169
|
+
License. However, in accepting such obligations, You may act only
|
|
170
|
+
on Your own behalf and on Your sole responsibility, not on behalf
|
|
171
|
+
of any other Contributor, and only if You agree to indemnify,
|
|
172
|
+
defend, and hold each Contributor harmless for any liability
|
|
173
|
+
incurred by, or claims asserted against, such Contributor by reason
|
|
174
|
+
of your accepting any such warranty or additional liability.
|
|
175
|
+
|
|
176
|
+
END OF TERMS AND CONDITIONS
|
|
177
|
+
|
|
178
|
+
Copyright 2024 10U Labs LLC
|
|
179
|
+
|
|
180
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
|
181
|
+
you may not use this file except in compliance with the License.
|
|
182
|
+
You may obtain a copy of the License at
|
|
183
|
+
|
|
184
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
185
|
+
|
|
186
|
+
Unless required by applicable law or agreed to in writing, software
|
|
187
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
|
188
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
189
|
+
See the License for the specific language governing permissions and
|
|
190
|
+
limitations under the License.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
assert_pytest_test_can_fail
|