streaming-claim-audit 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- streaming_claim_audit-0.1.0/LICENSE +21 -0
- streaming_claim_audit-0.1.0/MANIFEST.in +2 -0
- streaming_claim_audit-0.1.0/PKG-INFO +108 -0
- streaming_claim_audit-0.1.0/README.md +97 -0
- streaming_claim_audit-0.1.0/pyproject.toml +19 -0
- streaming_claim_audit-0.1.0/setup.cfg +4 -0
- streaming_claim_audit-0.1.0/src/streaming_claim_audit/__init__.py +2 -0
- streaming_claim_audit-0.1.0/src/streaming_claim_audit/__main__.py +44 -0
- streaming_claim_audit-0.1.0/src/streaming_claim_audit/audit.py +187 -0
- streaming_claim_audit-0.1.0/src/streaming_claim_audit.egg-info/PKG-INFO +108 -0
- streaming_claim_audit-0.1.0/src/streaming_claim_audit.egg-info/SOURCES.txt +20 -0
- streaming_claim_audit-0.1.0/src/streaming_claim_audit.egg-info/dependency_links.txt +1 -0
- streaming_claim_audit-0.1.0/src/streaming_claim_audit.egg-info/entry_points.txt +2 -0
- streaming_claim_audit-0.1.0/src/streaming_claim_audit.egg-info/top_level.txt +1 -0
- streaming_claim_audit-0.1.0/tests/fixtures/bad/README.md +1 -0
- streaming_claim_audit-0.1.0/tests/fixtures/bad/main.py +11 -0
- streaming_claim_audit-0.1.0/tests/fixtures/good/README.md +1 -0
- streaming_claim_audit-0.1.0/tests/fixtures/good/main.py +7 -0
- streaming_claim_audit-0.1.0/tests/fixtures/testonly/README.md +1 -0
- streaming_claim_audit-0.1.0/tests/fixtures/testonly/main.py +4 -0
- streaming_claim_audit-0.1.0/tests/fixtures/testonly/tests/test_read.py +5 -0
- streaming_claim_audit-0.1.0/tests/test_audit.py +245 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 streaming-claim-audit contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: streaming-claim-audit
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Offline static audit of documented streaming and low-memory claims
|
|
5
|
+
Author: streaming-claim-audit contributors
|
|
6
|
+
License: MIT
|
|
7
|
+
Requires-Python: >=3.9
|
|
8
|
+
Description-Content-Type: text/markdown
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Dynamic: license-file
|
|
11
|
+
|
|
12
|
+
# streaming-claim-audit
|
|
13
|
+
|
|
14
|
+
Version 0.1.0. An offline, stdlib-only CLI that compares documented streaming
|
|
15
|
+
and low-memory promises with static whole-input loading patterns in Python
|
|
16
|
+
source. It never executes audited code, makes network requests, or opens
|
|
17
|
+
`.env` files. No runtime dependencies. Requires Python 3.9 or newer.
|
|
18
|
+
|
|
19
|
+
## Install and use
|
|
20
|
+
|
|
21
|
+
```sh
|
|
22
|
+
python3 -m pip install .
|
|
23
|
+
streaming-claim-audit /path/to/project
|
|
24
|
+
python3 -m streaming_claim_audit /path/to/project --json
|
|
25
|
+
streaming-claim-audit --version
|
|
26
|
+
streaming-claim-audit tests/fixtures/testonly --fail-on HIGH
|
|
27
|
+
streaming-claim-audit . --verbose --ignore documented-streaming-claim-false
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
The path defaults to `.`. Exit codes: **0 PASS**, **2 FAIL**, **1 usage/input
|
|
31
|
+
error**. The default `--fail-on MEDIUM` fails for HIGH or MEDIUM;
|
|
32
|
+
`--fail-on HIGH` still prints MEDIUM findings but allows them to pass.
|
|
33
|
+
`--ignore RULE,...` removes named rules entirely. `--verbose` prints verified
|
|
34
|
+
claims. `--json` emits an object with `findings`, `verified_claims`, and
|
|
35
|
+
`summary`; each finding includes its documentation location and all evidence
|
|
36
|
+
locations. A verified claim means no matching evidence was found, not a proof.
|
|
37
|
+
|
|
38
|
+
## Scope and rules
|
|
39
|
+
|
|
40
|
+
Claims come only from root `README.md` and `docs/**/*.md`. Fenced code is
|
|
41
|
+
excluded; inline code counts. Negation suppresses the corresponding family
|
|
42
|
+
within a sentence. Streaming stems include stream, incremental, line-by-line,
|
|
43
|
+
lazy/lazily, iterator-based and yields. Low-memory stems include low memory,
|
|
44
|
+
constant memory, memory efficient, minimal memory and large file variants.
|
|
45
|
+
|
|
46
|
+
Production source includes recursive `*.py` files except `tests/`, `fixtures/`,
|
|
47
|
+
`docs/`, `examples/`, `test_*.py`, and `*_test.py`. Symlinked files and
|
|
48
|
+
directories are skipped, as are `.git`, `.venv`, `venv`, `__pycache__`, `build`,
|
|
49
|
+
and `dist` directories. Files and directories beginning with `.env` are never
|
|
50
|
+
opened. Invalid source syntax or unreadable inputs produce an input error.
|
|
51
|
+
|
|
52
|
+
Each family produces at most one finding, carrying every relevant evidence
|
|
53
|
+
item. Production evidence, or claims without production source, yields HIGH:
|
|
54
|
+
|
|
55
|
+
- `documented-streaming-claim-false`
|
|
56
|
+
- `documented-low-memory-claim-false`
|
|
57
|
+
|
|
58
|
+
Evidence confined to test/fixture code yields MEDIUM:
|
|
59
|
+
|
|
60
|
+
- `documented-streaming-claim-false-test-only`
|
|
61
|
+
- `documented-low-memory-claim-false-test-only`
|
|
62
|
+
|
|
63
|
+
Evidence includes whole-file `read_text`, `read_bytes`, unbounded file `read`,
|
|
64
|
+
`readlines`, file/list and CSV materialization, file-derived JSON loading and
|
|
65
|
+
pandas whole-frame call patterns. Direct file iteration, literal-size chunked
|
|
66
|
+
reads, line iteration with `json.loads(line)`, directly iterated CSV readers,
|
|
67
|
+
and generators do not count. No documented claims means PASS, regardless of
|
|
68
|
+
evidence. There are no reverse rules for undocumented streaming behavior.
|
|
69
|
+
|
|
70
|
+
## Differentiation
|
|
71
|
+
|
|
72
|
+
Memory profilers and tracemalloc observe runtime allocations; this tool checks
|
|
73
|
+
documentation promises against source without running the target. Bandit and
|
|
74
|
+
Ruff flag code patterns without relating them to those promises.
|
|
75
|
+
format-claim-audit checks which formats are supported, readonly-claim-audit
|
|
76
|
+
checks writes, and schema-claim-audit checks field names. None checks how much
|
|
77
|
+
input is held in memory. `wc` and file-size tools measure actual input bytes
|
|
78
|
+
rather than comparing documentation with implementation.
|
|
79
|
+
|
|
80
|
+
## Honest limitations
|
|
81
|
+
|
|
82
|
+
This is static pattern matching, not semantic judgement or a memory profiler.
|
|
83
|
+
Dead and unreachable code counts. Whole-file configuration reads may be small
|
|
84
|
+
and harmless, and a project may expose both streaming and non-streaming APIs;
|
|
85
|
+
findings require human review. Variable file objects and file-derived text are
|
|
86
|
+
tracked only through simple same-function assignments, not across calls,
|
|
87
|
+
containers or complex control flow. Import aliases are not generally resolved.
|
|
88
|
+
Pandas call-name matching (`pandas` and `pd`) does not verify imports; a
|
|
89
|
+
`chunksize` keyword suppresses it regardless of its value. Chunked-read detection
|
|
90
|
+
recognizes only literal nonnegative integer sizes; computed sizes may be flagged.
|
|
91
|
+
Arbitrary third-party loaders, wrappers, and memory growth outside the listed
|
|
92
|
+
patterns are not detected. Negation and sentence extraction are heuristic,
|
|
93
|
+
not a natural-language understanding system. No evidence is not proof of
|
|
94
|
+
bounded memory. This auditor itself loads source and documentation into memory.
|
|
95
|
+
|
|
96
|
+
## Development and validation
|
|
97
|
+
|
|
98
|
+
```sh
|
|
99
|
+
PYTHONPATH=src python3 -m unittest discover -s tests -v
|
|
100
|
+
python3 -m build
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
The sdist includes tests and fixtures via `MANIFEST.in`. The good fixture
|
|
104
|
+
passes with a verified STREAMING claim. The bad fixture produces exactly two
|
|
105
|
+
HIGH findings. The testonly fixture produces two MEDIUM findings and passes
|
|
106
|
+
only when `--fail-on HIGH` is used (or the relevant rules are ignored).
|
|
107
|
+
|
|
108
|
+
Licensed under the MIT License; see LICENSE.
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
# streaming-claim-audit
|
|
2
|
+
|
|
3
|
+
Version 0.1.0. An offline, stdlib-only CLI that compares documented streaming
|
|
4
|
+
and low-memory promises with static whole-input loading patterns in Python
|
|
5
|
+
source. It never executes audited code, makes network requests, or opens
|
|
6
|
+
`.env` files. No runtime dependencies. Requires Python 3.9 or newer.
|
|
7
|
+
|
|
8
|
+
## Install and use
|
|
9
|
+
|
|
10
|
+
```sh
|
|
11
|
+
python3 -m pip install .
|
|
12
|
+
streaming-claim-audit /path/to/project
|
|
13
|
+
python3 -m streaming_claim_audit /path/to/project --json
|
|
14
|
+
streaming-claim-audit --version
|
|
15
|
+
streaming-claim-audit tests/fixtures/testonly --fail-on HIGH
|
|
16
|
+
streaming-claim-audit . --verbose --ignore documented-streaming-claim-false
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
The path defaults to `.`. Exit codes: **0 PASS**, **2 FAIL**, **1 usage/input
|
|
20
|
+
error**. The default `--fail-on MEDIUM` fails for HIGH or MEDIUM;
|
|
21
|
+
`--fail-on HIGH` still prints MEDIUM findings but allows them to pass.
|
|
22
|
+
`--ignore RULE,...` removes named rules entirely. `--verbose` prints verified
|
|
23
|
+
claims. `--json` emits an object with `findings`, `verified_claims`, and
|
|
24
|
+
`summary`; each finding includes its documentation location and all evidence
|
|
25
|
+
locations. A verified claim means no matching evidence was found, not a proof.
|
|
26
|
+
|
|
27
|
+
## Scope and rules
|
|
28
|
+
|
|
29
|
+
Claims come only from root `README.md` and `docs/**/*.md`. Fenced code is
|
|
30
|
+
excluded; inline code counts. Negation suppresses the corresponding family
|
|
31
|
+
within a sentence. Streaming stems include stream, incremental, line-by-line,
|
|
32
|
+
lazy/lazily, iterator-based and yields. Low-memory stems include low memory,
|
|
33
|
+
constant memory, memory efficient, minimal memory and large file variants.
|
|
34
|
+
|
|
35
|
+
Production source includes recursive `*.py` files except `tests/`, `fixtures/`,
|
|
36
|
+
`docs/`, `examples/`, `test_*.py`, and `*_test.py`. Symlinked files and
|
|
37
|
+
directories are skipped, as are `.git`, `.venv`, `venv`, `__pycache__`, `build`,
|
|
38
|
+
and `dist` directories. Files and directories beginning with `.env` are never
|
|
39
|
+
opened. Invalid source syntax or unreadable inputs produce an input error.
|
|
40
|
+
|
|
41
|
+
Each family produces at most one finding, carrying every relevant evidence
|
|
42
|
+
item. Production evidence, or claims without production source, yields HIGH:
|
|
43
|
+
|
|
44
|
+
- `documented-streaming-claim-false`
|
|
45
|
+
- `documented-low-memory-claim-false`
|
|
46
|
+
|
|
47
|
+
Evidence confined to test/fixture code yields MEDIUM:
|
|
48
|
+
|
|
49
|
+
- `documented-streaming-claim-false-test-only`
|
|
50
|
+
- `documented-low-memory-claim-false-test-only`
|
|
51
|
+
|
|
52
|
+
Evidence includes whole-file `read_text`, `read_bytes`, unbounded file `read`,
|
|
53
|
+
`readlines`, file/list and CSV materialization, file-derived JSON loading and
|
|
54
|
+
pandas whole-frame call patterns. Direct file iteration, literal-size chunked
|
|
55
|
+
reads, line iteration with `json.loads(line)`, directly iterated CSV readers,
|
|
56
|
+
and generators do not count. No documented claims means PASS, regardless of
|
|
57
|
+
evidence. There are no reverse rules for undocumented streaming behavior.
|
|
58
|
+
|
|
59
|
+
## Differentiation
|
|
60
|
+
|
|
61
|
+
Memory profilers and tracemalloc observe runtime allocations; this tool checks
|
|
62
|
+
documentation promises against source without running the target. Bandit and
|
|
63
|
+
Ruff flag code patterns without relating them to those promises.
|
|
64
|
+
format-claim-audit checks which formats are supported, readonly-claim-audit
|
|
65
|
+
checks writes, and schema-claim-audit checks field names. None checks how much
|
|
66
|
+
input is held in memory. `wc` and file-size tools measure actual input bytes
|
|
67
|
+
rather than comparing documentation with implementation.
|
|
68
|
+
|
|
69
|
+
## Honest limitations
|
|
70
|
+
|
|
71
|
+
This is static pattern matching, not semantic judgement or a memory profiler.
|
|
72
|
+
Dead and unreachable code counts. Whole-file configuration reads may be small
|
|
73
|
+
and harmless, and a project may expose both streaming and non-streaming APIs;
|
|
74
|
+
findings require human review. Variable file objects and file-derived text are
|
|
75
|
+
tracked only through simple same-function assignments, not across calls,
|
|
76
|
+
containers or complex control flow. Import aliases are not generally resolved.
|
|
77
|
+
Pandas call-name matching (`pandas` and `pd`) does not verify imports; a
|
|
78
|
+
`chunksize` keyword suppresses it regardless of its value. Chunked-read detection
|
|
79
|
+
recognizes only literal nonnegative integer sizes; computed sizes may be flagged.
|
|
80
|
+
Arbitrary third-party loaders, wrappers, and memory growth outside the listed
|
|
81
|
+
patterns are not detected. Negation and sentence extraction are heuristic,
|
|
82
|
+
not a natural-language understanding system. No evidence is not proof of
|
|
83
|
+
bounded memory. This auditor itself loads source and documentation into memory.
|
|
84
|
+
|
|
85
|
+
## Development and validation
|
|
86
|
+
|
|
87
|
+
```sh
|
|
88
|
+
PYTHONPATH=src python3 -m unittest discover -s tests -v
|
|
89
|
+
python3 -m build
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
The sdist includes tests and fixtures via `MANIFEST.in`. The good fixture
|
|
93
|
+
passes with a verified STREAMING claim. The bad fixture produces exactly two
|
|
94
|
+
HIGH findings. The testonly fixture produces two MEDIUM findings and passes
|
|
95
|
+
only when `--fail-on HIGH` is used (or the relevant rules are ignored).
|
|
96
|
+
|
|
97
|
+
Licensed under the MIT License; see LICENSE.
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=61"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "streaming-claim-audit"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Offline static audit of documented streaming and low-memory claims"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.9"
|
|
11
|
+
license = {text = "MIT"}
|
|
12
|
+
authors = [{name = "streaming-claim-audit contributors"}]
|
|
13
|
+
dependencies = []
|
|
14
|
+
|
|
15
|
+
[project.scripts]
|
|
16
|
+
streaming-claim-audit = "streaming_claim_audit.__main__:main"
|
|
17
|
+
|
|
18
|
+
[tool.setuptools.packages.find]
|
|
19
|
+
where = ["src"]
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
"""Command-line interface; audited code is never imported or executed."""
|
|
2
|
+
import argparse
|
|
3
|
+
import json
|
|
4
|
+
import sys
|
|
5
|
+
from . import __version__
|
|
6
|
+
from .audit import audit
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class Parser(argparse.ArgumentParser):
|
|
10
|
+
def error(self, message):
|
|
11
|
+
self.print_usage(sys.stderr)
|
|
12
|
+
self.exit(1, self.prog + ': error: ' + message + '\n')
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def main(argv=None):
|
|
16
|
+
parser = Parser(prog='streaming-claim-audit')
|
|
17
|
+
parser.add_argument('path', nargs='?', default='.')
|
|
18
|
+
parser.add_argument('--version', action='version', version=__version__)
|
|
19
|
+
parser.add_argument('--json', action='store_true')
|
|
20
|
+
parser.add_argument('--verbose', action='store_true')
|
|
21
|
+
parser.add_argument('--ignore', default='', metavar='RULE,...')
|
|
22
|
+
parser.add_argument('--fail-on', choices=('MEDIUM', 'HIGH'), default='MEDIUM')
|
|
23
|
+
args = parser.parse_args(argv)
|
|
24
|
+
try:
|
|
25
|
+
result = audit(args.path, {r.strip() for r in args.ignore.split(',')}, args.fail_on)
|
|
26
|
+
except (OSError, ValueError, SyntaxError, UnicodeError) as error:
|
|
27
|
+
print('streaming-claim-audit: error: ' + str(error), file=sys.stderr)
|
|
28
|
+
return 1
|
|
29
|
+
if args.json:
|
|
30
|
+
print(json.dumps(result, indent=2))
|
|
31
|
+
else:
|
|
32
|
+
for finding in result['findings']:
|
|
33
|
+
print('{severity} {rule} {file}:{line} {message}'.format(**finding))
|
|
34
|
+
for item in finding['evidence']:
|
|
35
|
+
print(' {file}:{line} {kind}'.format(**item))
|
|
36
|
+
if args.verbose:
|
|
37
|
+
for claim in result['verified_claims']:
|
|
38
|
+
print('VERIFIED {family} {file}:{line}'.format(**claim))
|
|
39
|
+
print(result['summary']['status'])
|
|
40
|
+
return 2 if result['summary']['status'] == 'FAIL' else 0
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
if __name__ == '__main__':
|
|
44
|
+
sys.exit(main())
|
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
"""Conservative AST evidence and Markdown claim extraction."""
|
|
2
|
+
import ast
|
|
3
|
+
import os
|
|
4
|
+
import re
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
PATTERNS = {
|
|
8
|
+
'STREAMING': r'\b(?:stream\w*|incremental\w*|line[ -]by[ -]line|lazy|lazily|iterator[ -]based|yields)\b',
|
|
9
|
+
'LOW_MEMORY': r'\b(?:low[ -]memory|constant[ -]memory|memory[ -]efficient|minimal memory|large[ -]files?)\b',
|
|
10
|
+
}
|
|
11
|
+
RULES = {family: 'documented-' + family.lower().replace('_', '-') + '-claim-false'
|
|
12
|
+
for family in PATTERNS}
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def claims(text, filename='README.md'):
|
|
16
|
+
"""Return one location per family; ignore fenced examples and negations."""
|
|
17
|
+
result = {}
|
|
18
|
+
fence = None
|
|
19
|
+
prose = []
|
|
20
|
+
for number, line in enumerate(text.splitlines(), 1):
|
|
21
|
+
marker = re.match(r'^\s{0,3}(`{3,}|~{3,})', line)
|
|
22
|
+
if marker:
|
|
23
|
+
token = marker.group(1)
|
|
24
|
+
if fence is None:
|
|
25
|
+
fence = token
|
|
26
|
+
elif token[0] == fence[0] and len(token) >= len(fence):
|
|
27
|
+
fence = None
|
|
28
|
+
prose.append('')
|
|
29
|
+
else:
|
|
30
|
+
prose.append(line if fence is None else '')
|
|
31
|
+
joined = '\n'.join(prose)
|
|
32
|
+
for match in re.finditer(r'[^.!?]+(?:[.!?]|$)', joined):
|
|
33
|
+
sentence = match.group()
|
|
34
|
+
normalized = re.sub(r'[`*_]', '', sentence.lower()).replace('-', ' ')
|
|
35
|
+
for family, pattern in PATTERNS.items():
|
|
36
|
+
hit = re.search(pattern, sentence, re.I)
|
|
37
|
+
if not hit:
|
|
38
|
+
continue
|
|
39
|
+
# Negation attached to a family phrase suppresses that family only.
|
|
40
|
+
family_pattern = pattern.replace('[ -]', ' ')
|
|
41
|
+
negated = re.search(
|
|
42
|
+
r'\b(?:not|never|no|without|does not|do not|is not)\s+'
|
|
43
|
+
r'(?:(?:support|use|provide|offer|guarantee|a|any)\s+){0,3}'
|
|
44
|
+
+ family_pattern, normalized, re.I)
|
|
45
|
+
if not negated:
|
|
46
|
+
line = joined.count('\n', 0, match.start() + hit.start()) + 1
|
|
47
|
+
result.setdefault(family, {'family': family, 'file': filename, 'line': line})
|
|
48
|
+
return result
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def name(node):
|
|
52
|
+
if isinstance(node, ast.Name):
|
|
53
|
+
return node.id
|
|
54
|
+
if isinstance(node, ast.Attribute):
|
|
55
|
+
return name(node.value) + '.' + node.attr
|
|
56
|
+
return ''
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class Evidence(ast.NodeVisitor):
|
|
60
|
+
def __init__(self, filename):
|
|
61
|
+
self.filename = filename
|
|
62
|
+
self.flow = {}
|
|
63
|
+
self.items = []
|
|
64
|
+
|
|
65
|
+
def kind(self, node):
|
|
66
|
+
if isinstance(node, ast.Name):
|
|
67
|
+
return self.flow.get(node.id)
|
|
68
|
+
if isinstance(node, ast.Call):
|
|
69
|
+
call = name(node.func)
|
|
70
|
+
if call in ('open', 'io.open') or call.endswith('.open'):
|
|
71
|
+
return 'file'
|
|
72
|
+
if call in ('Path', 'pathlib.Path'):
|
|
73
|
+
return 'path'
|
|
74
|
+
if self.reason(node):
|
|
75
|
+
return 'whole'
|
|
76
|
+
return None
|
|
77
|
+
|
|
78
|
+
def reason(self, node):
|
|
79
|
+
call = name(node.func)
|
|
80
|
+
attr = node.func.attr if isinstance(node.func, ast.Attribute) else ''
|
|
81
|
+
receiver = node.func.value if isinstance(node.func, ast.Attribute) else None
|
|
82
|
+
if attr in ('read_text', 'read_bytes', 'readlines'):
|
|
83
|
+
return attr
|
|
84
|
+
if attr == 'read' and self.kind(receiver) == 'file':
|
|
85
|
+
size = node.args[0] if node.args else next(
|
|
86
|
+
(k.value for k in node.keywords if k.arg == 'size'), None)
|
|
87
|
+
if not (isinstance(size, ast.Constant) and type(size.value) is int and size.value >= 0):
|
|
88
|
+
return 'unbounded read'
|
|
89
|
+
if call == 'json.load' and node.args and self.kind(node.args[0]) == 'file':
|
|
90
|
+
return 'json.load(file)'
|
|
91
|
+
if call == 'json.loads' and node.args and self.kind(node.args[0]) == 'whole':
|
|
92
|
+
return 'json.loads(whole-file text)'
|
|
93
|
+
if call == 'list' and node.args:
|
|
94
|
+
arg = node.args[0]
|
|
95
|
+
if self.kind(arg) == 'file' or (isinstance(arg, ast.Call) and name(arg.func) in ('csv.reader', 'csv.DictReader')):
|
|
96
|
+
return 'materialized file iterator'
|
|
97
|
+
if call == 'csv.reader' and node.args and isinstance(node.args[0], ast.Call) and name(node.args[0].func) == 'list':
|
|
98
|
+
return 'csv.reader(list(...))'
|
|
99
|
+
if call in ('pandas.read_csv', 'pandas.read_json', 'pandas.read_parquet', 'pd.read_csv', 'pd.read_json', 'pd.read_parquet') and not any(k.arg == 'chunksize' for k in node.keywords):
|
|
100
|
+
return 'whole-frame load'
|
|
101
|
+
return None
|
|
102
|
+
|
|
103
|
+
def visit_Call(self, node):
|
|
104
|
+
reason = self.reason(node)
|
|
105
|
+
if reason:
|
|
106
|
+
self.items.append({'file': self.filename, 'line': node.lineno, 'kind': reason})
|
|
107
|
+
self.generic_visit(node)
|
|
108
|
+
|
|
109
|
+
def visit_Assign(self, node):
|
|
110
|
+
self.visit(node.value)
|
|
111
|
+
for target in node.targets:
|
|
112
|
+
if isinstance(target, ast.Name):
|
|
113
|
+
self.flow[target.id] = self.kind(node.value)
|
|
114
|
+
|
|
115
|
+
def visit_AnnAssign(self, node):
|
|
116
|
+
if node.value:
|
|
117
|
+
self.visit(node.value)
|
|
118
|
+
if isinstance(node.target, ast.Name):
|
|
119
|
+
self.flow[node.target.id] = self.kind(node.value)
|
|
120
|
+
|
|
121
|
+
def visit_With(self, node):
|
|
122
|
+
for item in node.items:
|
|
123
|
+
self.visit(item.context_expr)
|
|
124
|
+
if isinstance(item.optional_vars, ast.Name):
|
|
125
|
+
self.flow[item.optional_vars.id] = self.kind(item.context_expr)
|
|
126
|
+
for statement in node.body:
|
|
127
|
+
self.visit(statement)
|
|
128
|
+
|
|
129
|
+
def visit_FunctionDef(self, node):
|
|
130
|
+
previous = self.flow
|
|
131
|
+
self.flow = {}
|
|
132
|
+
for statement in node.body:
|
|
133
|
+
self.visit(statement)
|
|
134
|
+
self.flow = previous
|
|
135
|
+
|
|
136
|
+
visit_AsyncFunctionDef = visit_FunctionDef
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def evidence(text, filename='main.py'):
|
|
140
|
+
visitor = Evidence(filename)
|
|
141
|
+
visitor.visit(ast.parse(text, filename=filename))
|
|
142
|
+
return sorted(visitor.items, key=lambda item: (item['line'], item['kind']))
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def audit(root, ignore=(), fail_on='MEDIUM'):
|
|
146
|
+
root = Path(root)
|
|
147
|
+
if not root.is_dir():
|
|
148
|
+
raise ValueError('audit path must be an existing directory')
|
|
149
|
+
found = {}
|
|
150
|
+
production, test_items = [], []
|
|
151
|
+
source_count = 0
|
|
152
|
+
for directory, dirs, files in os.walk(root, onerror=lambda error: (_ for _ in ()).throw(error)):
|
|
153
|
+
dirs[:] = sorted(d for d in dirs if d not in ('.git', '.venv', 'venv', '__pycache__', 'build', 'dist') and not d.startswith('.env') and not (Path(directory) / d).is_symlink())
|
|
154
|
+
for filename in sorted(files):
|
|
155
|
+
path = Path(directory) / filename
|
|
156
|
+
relative = path.relative_to(root)
|
|
157
|
+
parts = relative.parts
|
|
158
|
+
if filename.startswith('.env') or path.is_symlink():
|
|
159
|
+
continue
|
|
160
|
+
if relative.as_posix() == 'README.md' or (parts[0] == 'docs' and path.suffix == '.md'):
|
|
161
|
+
for family, claim in claims(path.read_text(encoding='utf-8'), relative.as_posix()).items():
|
|
162
|
+
found.setdefault(family, claim)
|
|
163
|
+
if path.suffix != '.py' or any(p in ('docs', 'examples') for p in parts[:-1]):
|
|
164
|
+
continue
|
|
165
|
+
is_test = any(p in ('tests', 'fixtures') for p in parts[:-1]) or filename.startswith('test_') or filename.endswith('_test.py')
|
|
166
|
+
items = evidence(path.read_text(encoding='utf-8'), relative.as_posix())
|
|
167
|
+
if is_test:
|
|
168
|
+
test_items.extend(items)
|
|
169
|
+
else:
|
|
170
|
+
source_count += 1
|
|
171
|
+
production.extend(items)
|
|
172
|
+
findings, verified = [], []
|
|
173
|
+
for family, claim in sorted(found.items()):
|
|
174
|
+
severity = 'HIGH' if production or not source_count else 'MEDIUM'
|
|
175
|
+
items = production if production else test_items
|
|
176
|
+
if not source_count or items:
|
|
177
|
+
rule = RULES[family] + ('-test-only' if severity == 'MEDIUM' else '')
|
|
178
|
+
if rule not in ignore:
|
|
179
|
+
findings.append(dict(claim, severity=severity, rule=rule, evidence=items,
|
|
180
|
+
message='No production Python source found.' if not source_count else 'Whole-input loading contradicts documented claim.'))
|
|
181
|
+
else:
|
|
182
|
+
verified.append(claim)
|
|
183
|
+
failed = any(f['severity'] == 'HIGH' or fail_on == 'MEDIUM' for f in findings)
|
|
184
|
+
return {'findings': findings, 'verified_claims': verified,
|
|
185
|
+
'summary': {'status': 'FAIL' if failed else 'PASS', 'findings': len(findings),
|
|
186
|
+
'high': sum(f['severity'] == 'HIGH' for f in findings),
|
|
187
|
+
'medium': sum(f['severity'] == 'MEDIUM' for f in findings)}}
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: streaming-claim-audit
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Offline static audit of documented streaming and low-memory claims
|
|
5
|
+
Author: streaming-claim-audit contributors
|
|
6
|
+
License: MIT
|
|
7
|
+
Requires-Python: >=3.9
|
|
8
|
+
Description-Content-Type: text/markdown
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Dynamic: license-file
|
|
11
|
+
|
|
12
|
+
# streaming-claim-audit
|
|
13
|
+
|
|
14
|
+
Version 0.1.0. An offline, stdlib-only CLI that compares documented streaming
|
|
15
|
+
and low-memory promises with static whole-input loading patterns in Python
|
|
16
|
+
source. It never executes audited code, makes network requests, or opens
|
|
17
|
+
`.env` files. No runtime dependencies. Requires Python 3.9 or newer.
|
|
18
|
+
|
|
19
|
+
## Install and use
|
|
20
|
+
|
|
21
|
+
```sh
|
|
22
|
+
python3 -m pip install .
|
|
23
|
+
streaming-claim-audit /path/to/project
|
|
24
|
+
python3 -m streaming_claim_audit /path/to/project --json
|
|
25
|
+
streaming-claim-audit --version
|
|
26
|
+
streaming-claim-audit tests/fixtures/testonly --fail-on HIGH
|
|
27
|
+
streaming-claim-audit . --verbose --ignore documented-streaming-claim-false
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
The path defaults to `.`. Exit codes: **0 PASS**, **2 FAIL**, **1 usage/input
|
|
31
|
+
error**. The default `--fail-on MEDIUM` fails for HIGH or MEDIUM;
|
|
32
|
+
`--fail-on HIGH` still prints MEDIUM findings but allows them to pass.
|
|
33
|
+
`--ignore RULE,...` removes named rules entirely. `--verbose` prints verified
|
|
34
|
+
claims. `--json` emits an object with `findings`, `verified_claims`, and
|
|
35
|
+
`summary`; each finding includes its documentation location and all evidence
|
|
36
|
+
locations. A verified claim means no matching evidence was found, not a proof.
|
|
37
|
+
|
|
38
|
+
## Scope and rules
|
|
39
|
+
|
|
40
|
+
Claims come only from root `README.md` and `docs/**/*.md`. Fenced code is
|
|
41
|
+
excluded; inline code counts. Negation suppresses the corresponding family
|
|
42
|
+
within a sentence. Streaming stems include stream, incremental, line-by-line,
|
|
43
|
+
lazy/lazily, iterator-based and yields. Low-memory stems include low memory,
|
|
44
|
+
constant memory, memory efficient, minimal memory and large file variants.
|
|
45
|
+
|
|
46
|
+
Production source includes recursive `*.py` files except `tests/`, `fixtures/`,
|
|
47
|
+
`docs/`, `examples/`, `test_*.py`, and `*_test.py`. Symlinked files and
|
|
48
|
+
directories are skipped, as are `.git`, `.venv`, `venv`, `__pycache__`, `build`,
|
|
49
|
+
and `dist` directories. Files and directories beginning with `.env` are never
|
|
50
|
+
opened. Invalid source syntax or unreadable inputs produce an input error.
|
|
51
|
+
|
|
52
|
+
Each family produces at most one finding, carrying every relevant evidence
|
|
53
|
+
item. Production evidence, or claims without production source, yields HIGH:
|
|
54
|
+
|
|
55
|
+
- `documented-streaming-claim-false`
|
|
56
|
+
- `documented-low-memory-claim-false`
|
|
57
|
+
|
|
58
|
+
Evidence confined to test/fixture code yields MEDIUM:
|
|
59
|
+
|
|
60
|
+
- `documented-streaming-claim-false-test-only`
|
|
61
|
+
- `documented-low-memory-claim-false-test-only`
|
|
62
|
+
|
|
63
|
+
Evidence includes whole-file `read_text`, `read_bytes`, unbounded file `read`,
|
|
64
|
+
`readlines`, file/list and CSV materialization, file-derived JSON loading and
|
|
65
|
+
pandas whole-frame call patterns. Direct file iteration, literal-size chunked
|
|
66
|
+
reads, line iteration with `json.loads(line)`, directly iterated CSV readers,
|
|
67
|
+
and generators do not count. No documented claims means PASS, regardless of
|
|
68
|
+
evidence. There are no reverse rules for undocumented streaming behavior.
|
|
69
|
+
|
|
70
|
+
## Differentiation
|
|
71
|
+
|
|
72
|
+
Memory profilers and tracemalloc observe runtime allocations; this tool checks
|
|
73
|
+
documentation promises against source without running the target. Bandit and
|
|
74
|
+
Ruff flag code patterns without relating them to those promises.
|
|
75
|
+
format-claim-audit checks which formats are supported, readonly-claim-audit
|
|
76
|
+
checks writes, and schema-claim-audit checks field names. None checks how much
|
|
77
|
+
input is held in memory. `wc` and file-size tools measure actual input bytes
|
|
78
|
+
rather than comparing documentation with implementation.
|
|
79
|
+
|
|
80
|
+
## Honest limitations
|
|
81
|
+
|
|
82
|
+
This is static pattern matching, not semantic judgement or a memory profiler.
|
|
83
|
+
Dead and unreachable code counts. Whole-file configuration reads may be small
|
|
84
|
+
and harmless, and a project may expose both streaming and non-streaming APIs;
|
|
85
|
+
findings require human review. Variable file objects and file-derived text are
|
|
86
|
+
tracked only through simple same-function assignments, not across calls,
|
|
87
|
+
containers or complex control flow. Import aliases are not generally resolved.
|
|
88
|
+
Pandas call-name matching (`pandas` and `pd`) does not verify imports; a
|
|
89
|
+
`chunksize` keyword suppresses it regardless of its value. Chunked-read detection
|
|
90
|
+
recognizes only literal nonnegative integer sizes; computed sizes may be flagged.
|
|
91
|
+
Arbitrary third-party loaders, wrappers, and memory growth outside the listed
|
|
92
|
+
patterns are not detected. Negation and sentence extraction are heuristic,
|
|
93
|
+
not a natural-language understanding system. No evidence is not proof of
|
|
94
|
+
bounded memory. This auditor itself loads source and documentation into memory.
|
|
95
|
+
|
|
96
|
+
## Development and validation
|
|
97
|
+
|
|
98
|
+
```sh
|
|
99
|
+
PYTHONPATH=src python3 -m unittest discover -s tests -v
|
|
100
|
+
python3 -m build
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
The sdist includes tests and fixtures via `MANIFEST.in`. The good fixture
|
|
104
|
+
passes with a verified STREAMING claim. The bad fixture produces exactly two
|
|
105
|
+
HIGH findings. The testonly fixture produces two MEDIUM findings and passes
|
|
106
|
+
only when `--fail-on HIGH` is used (or the relevant rules are ignored).
|
|
107
|
+
|
|
108
|
+
Licensed under the MIT License; see LICENSE.
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
MANIFEST.in
|
|
3
|
+
README.md
|
|
4
|
+
pyproject.toml
|
|
5
|
+
src/streaming_claim_audit/__init__.py
|
|
6
|
+
src/streaming_claim_audit/__main__.py
|
|
7
|
+
src/streaming_claim_audit/audit.py
|
|
8
|
+
src/streaming_claim_audit.egg-info/PKG-INFO
|
|
9
|
+
src/streaming_claim_audit.egg-info/SOURCES.txt
|
|
10
|
+
src/streaming_claim_audit.egg-info/dependency_links.txt
|
|
11
|
+
src/streaming_claim_audit.egg-info/entry_points.txt
|
|
12
|
+
src/streaming_claim_audit.egg-info/top_level.txt
|
|
13
|
+
tests/test_audit.py
|
|
14
|
+
tests/fixtures/bad/README.md
|
|
15
|
+
tests/fixtures/bad/main.py
|
|
16
|
+
tests/fixtures/good/README.md
|
|
17
|
+
tests/fixtures/good/main.py
|
|
18
|
+
tests/fixtures/testonly/README.md
|
|
19
|
+
tests/fixtures/testonly/main.py
|
|
20
|
+
tests/fixtures/testonly/tests/test_read.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
streaming_claim_audit
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
This tool provides streaming, constant memory, and handles large files.
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import json
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
def records(path):
|
|
6
|
+
text = Path(path).read_text(encoding='utf-8')
|
|
7
|
+
with open(path, encoding='utf-8') as f:
|
|
8
|
+
data = f.read()
|
|
9
|
+
with open(path, encoding='utf-8') as f:
|
|
10
|
+
parsed = json.load(f)
|
|
11
|
+
return text, data, parsed
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
This tool streams large JSONL files line by line.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
This tool provides streaming, constant memory, and handles large files.
|
|
@@ -0,0 +1,245 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import os
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
import subprocess
|
|
5
|
+
import sys
|
|
6
|
+
import tempfile
|
|
7
|
+
import unittest
|
|
8
|
+
from unittest.mock import patch
|
|
9
|
+
|
|
10
|
+
from streaming_claim_audit.audit import audit, claims, evidence, RULES
|
|
11
|
+
|
|
12
|
+
FIXTURES = Path(__file__).parent / 'fixtures'
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class ClaimTests(unittest.TestCase):
|
|
16
|
+
pass
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
CLAIMS = [
|
|
20
|
+
('streaming', 'STREAMING'), ('streams', 'STREAMING'),
|
|
21
|
+
('stream-based', 'STREAMING'), ('stream processing', 'STREAMING'),
|
|
22
|
+
('processes records as a stream', 'STREAMING'),
|
|
23
|
+
('reads data incrementally', 'STREAMING'), ('line-by-line', 'STREAMING'),
|
|
24
|
+
('line by line', 'STREAMING'), ('lazily', 'STREAMING'),
|
|
25
|
+
('lazy loading', 'STREAMING'), ('iterator-based', 'STREAMING'),
|
|
26
|
+
('yields records one at a time', 'STREAMING'),
|
|
27
|
+
('low-memory', 'LOW_MEMORY'), ('low memory', 'LOW_MEMORY'),
|
|
28
|
+
('constant-memory', 'LOW_MEMORY'), ('constant memory', 'LOW_MEMORY'),
|
|
29
|
+
('memory-efficient', 'LOW_MEMORY'), ('memory efficient', 'LOW_MEMORY'),
|
|
30
|
+
('minimal memory', 'LOW_MEMORY'), ('handles large files', 'LOW_MEMORY'),
|
|
31
|
+
('scales to large files', 'LOW_MEMORY'), ('large-file support', 'LOW_MEMORY'),
|
|
32
|
+
('STREAMING', 'STREAMING'), ('`streaming`', 'STREAMING'),
|
|
33
|
+
]
|
|
34
|
+
for index, (phrase, family) in enumerate(CLAIMS):
|
|
35
|
+
def check(self, phrase=phrase, family=family):
|
|
36
|
+
self.assertIn(family, claims(phrase))
|
|
37
|
+
setattr(ClaimTests, 'test_claim_%02d' % index, check)
|
|
38
|
+
|
|
39
|
+
SUPPRESSED = ['not streaming', 'does not stream', 'no streaming support',
|
|
40
|
+
'not low-memory', 'no constant memory', 'without lazy loading',
|
|
41
|
+
'```\nstreaming\n```', '~~~md\nlow memory\n~~~',
|
|
42
|
+
'An ordinary command-line utility.']
|
|
43
|
+
for index, phrase in enumerate(SUPPRESSED):
|
|
44
|
+
def check(self, phrase=phrase):
|
|
45
|
+
self.assertEqual(claims(phrase), {})
|
|
46
|
+
setattr(ClaimTests, 'test_suppression_%02d' % index, check)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
class EvidenceTests(unittest.TestCase):
|
|
50
|
+
pass
|
|
51
|
+
|
|
52
|
+
POSITIVE = [
|
|
53
|
+
'Path(p).read_text()', 'Path(p).read_bytes()',
|
|
54
|
+
'f = open(p)\nf.read()', 'f = io.open(p)\nf.read()',
|
|
55
|
+
'with open(p) as f:\n f.read()',
|
|
56
|
+
'with Path(p).open() as f:\n f.read()',
|
|
57
|
+
'f = open(p)\nf.readlines()', 'x.readlines()',
|
|
58
|
+
'f = open(p)\nlist(f)', 'f = open(p)\njson.load(f)',
|
|
59
|
+
'f = open(p)\ndata = f.read()\njson.loads(data)',
|
|
60
|
+
'data = Path(p).read_text()\njson.loads(data)',
|
|
61
|
+
'csv.reader(list(rows))', 'list(csv.reader(f))',
|
|
62
|
+
'list(csv.DictReader(f))', 'pandas.read_csv(p)',
|
|
63
|
+
'pandas.read_json(p)', 'pandas.read_parquet(p)', 'pd.read_csv(p)',
|
|
64
|
+
'f = open(p)\nf.read().splitlines()',
|
|
65
|
+
'if False:\n Path(p).read_text()',
|
|
66
|
+
'f = open(p)\nf.read(-1)', 'f = open(p)\nf.read(size=n)',
|
|
67
|
+
'f = open(p)\ng = f\ng.read()',
|
|
68
|
+
]
|
|
69
|
+
NEGATIVE = [
|
|
70
|
+
'with open(p) as f:\n for line in f:\n print(line)',
|
|
71
|
+
'with open(p) as f:\n while True:\n line = f.readline()\n if not line: break',
|
|
72
|
+
'f = open(p)\nf.read(4096)', 'f = open(p)\nf.read(size=4096)',
|
|
73
|
+
'with open(p) as f:\n for line in f:\n json.loads(line)',
|
|
74
|
+
'for row in csv.reader(f):\n print(row)',
|
|
75
|
+
'for row in csv.DictReader(f):\n print(row)',
|
|
76
|
+
'f = open(p)\niter(f)', 'f = open(p)\n(x for x in f)',
|
|
77
|
+
'with Path(p).open() as f:\n for line in f:\n print(line)',
|
|
78
|
+
'with Path(p).open() as f:\n f.read(4096)',
|
|
79
|
+
'pandas.read_csv(p, chunksize=100)',
|
|
80
|
+
'pandas.read_json(p, chunksize=100)',
|
|
81
|
+
'pandas.read_parquet(p, chunksize=100)',
|
|
82
|
+
'def parse(text):\n return json.loads(text)',
|
|
83
|
+
'def a():\n f = open(p)\ndef b():\n f.read()',
|
|
84
|
+
'f = open(p)\nf = other\nf.read()',
|
|
85
|
+
]
|
|
86
|
+
for positive, snippets in ((True, POSITIVE), (False, NEGATIVE)):
|
|
87
|
+
for index, source in enumerate(snippets):
|
|
88
|
+
def check(self, source=source, positive=positive):
|
|
89
|
+
self.assertEqual(bool(evidence(source)), positive)
|
|
90
|
+
setattr(EvidenceTests, 'test_%s_%02d' % ('read_all' if positive else 'excluded', index), check)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
class AuditTests(unittest.TestCase):
|
|
94
|
+
def project(self, readme='streaming, constant memory', source='for line in open(p):\n print(line)'):
|
|
95
|
+
temporary = tempfile.TemporaryDirectory()
|
|
96
|
+
self.addCleanup(temporary.cleanup)
|
|
97
|
+
root = Path(temporary.name)
|
|
98
|
+
(root / 'README.md').write_text(readme, encoding='utf-8')
|
|
99
|
+
if source is not None:
|
|
100
|
+
(root / 'main.py').write_text(source, encoding='utf-8')
|
|
101
|
+
return root
|
|
102
|
+
|
|
103
|
+
def test_good(self):
|
|
104
|
+
result = audit(FIXTURES / 'good')
|
|
105
|
+
self.assertEqual(result['summary']['status'], 'PASS')
|
|
106
|
+
self.assertIn('STREAMING', [c['family'] for c in result['verified_claims']])
|
|
107
|
+
|
|
108
|
+
def test_bad_grouping(self):
|
|
109
|
+
result = audit(FIXTURES / 'bad')
|
|
110
|
+
self.assertEqual(result['summary']['high'], 2)
|
|
111
|
+
for finding in result['findings']:
|
|
112
|
+
self.assertEqual(len(finding['evidence']), 3)
|
|
113
|
+
self.assertEqual([i['line'] for i in finding['evidence']], [6, 8, 10])
|
|
114
|
+
|
|
115
|
+
def test_testonly(self):
|
|
116
|
+
result = audit(FIXTURES / 'testonly')
|
|
117
|
+
self.assertEqual(result['summary']['medium'], 2)
|
|
118
|
+
self.assertEqual(result['summary']['status'], 'FAIL')
|
|
119
|
+
|
|
120
|
+
def test_threshold(self):
|
|
121
|
+
result = audit(FIXTURES / 'testonly', fail_on='HIGH')
|
|
122
|
+
self.assertEqual(result['summary']['status'], 'PASS')
|
|
123
|
+
self.assertEqual(len(result['findings']), 2)
|
|
124
|
+
|
|
125
|
+
def test_ignore(self):
|
|
126
|
+
result = audit(FIXTURES / 'bad', ignore=RULES.values())
|
|
127
|
+
self.assertEqual(result['findings'], [])
|
|
128
|
+
self.assertEqual(result['summary']['status'], 'PASS')
|
|
129
|
+
|
|
130
|
+
def test_no_claims(self):
|
|
131
|
+
self.assertEqual(audit(self.project('ordinary utility', 'Path(p).read_text()'))['summary']['status'], 'PASS')
|
|
132
|
+
|
|
133
|
+
def test_no_source(self):
|
|
134
|
+
result = audit(self.project(source=None))
|
|
135
|
+
self.assertEqual(result['summary']['high'], 2)
|
|
136
|
+
self.assertTrue(all(not f['evidence'] for f in result['findings']))
|
|
137
|
+
|
|
138
|
+
def test_docs_nested(self):
|
|
139
|
+
root = self.project('ordinary utility', 'Path(p).read_text()')
|
|
140
|
+
(root / 'docs' / 'nested').mkdir(parents=True)
|
|
141
|
+
(root / 'docs' / 'nested' / 'guide.md').write_text('streaming', encoding='utf-8')
|
|
142
|
+
self.assertEqual(audit(root)['summary']['high'], 1)
|
|
143
|
+
|
|
144
|
+
def test_other_markdown_ignored(self):
|
|
145
|
+
root = self.project('ordinary utility')
|
|
146
|
+
(root / 'OTHER.md').write_text('streaming', encoding='utf-8')
|
|
147
|
+
self.assertEqual(audit(root)['verified_claims'], [])
|
|
148
|
+
|
|
149
|
+
def test_symlink(self):
|
|
150
|
+
root = self.project()
|
|
151
|
+
(root / 'linked.py').symlink_to(FIXTURES / 'bad' / 'main.py')
|
|
152
|
+
(root / 'linked').symlink_to(FIXTURES / 'bad', target_is_directory=True)
|
|
153
|
+
self.assertEqual(audit(root)['summary']['status'], 'PASS')
|
|
154
|
+
|
|
155
|
+
def test_unreadable_root(self):
|
|
156
|
+
with patch('streaming_claim_audit.audit.os.walk', side_effect=PermissionError('denied')):
|
|
157
|
+
with self.assertRaises(PermissionError):
|
|
158
|
+
audit(self.project())
|
|
159
|
+
|
|
160
|
+
def test_invalid_syntax(self):
|
|
161
|
+
with self.assertRaises(SyntaxError):
|
|
162
|
+
audit(self.project(source='def !'))
|
|
163
|
+
|
|
164
|
+
def test_family_specific_negation(self):
|
|
165
|
+
self.assertEqual(set(claims('Not streaming but constant memory.')), {'LOW_MEMORY'})
|
|
166
|
+
|
|
167
|
+
def test_separate_sentences(self):
|
|
168
|
+
self.assertIn('STREAMING', claims('Not streaming. Another API streams data.'))
|
|
169
|
+
|
|
170
|
+
def test_fence_location(self):
|
|
171
|
+
self.assertEqual(claims('```\nstreaming\n```\nStreams data.')['STREAMING']['line'], 4)
|
|
172
|
+
|
|
173
|
+
def test_function_flow(self):
|
|
174
|
+
items = evidence('def load(p):\n with open(p) as f:\n data = f.read()\n return json.loads(data)')
|
|
175
|
+
self.assertEqual(len(items), 2)
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
for index, location in enumerate(('tests/helper.py', 'fixtures/helper.py', 'test_helper.py', 'helper_test.py', 'docs/helper.py', 'examples/helper.py')):
|
|
179
|
+
def check(self, location=location):
|
|
180
|
+
root = self.project()
|
|
181
|
+
path = root / location
|
|
182
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
183
|
+
path.write_text('Path(p).read_text()', encoding='utf-8')
|
|
184
|
+
result = audit(root)
|
|
185
|
+
self.assertEqual(result['summary']['high'], 0)
|
|
186
|
+
self.assertEqual(result['summary']['medium'], 0 if location.startswith(('docs/', 'examples/')) else 2)
|
|
187
|
+
setattr(AuditTests, 'test_source_exclusion_%02d' % index, check)
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
class CLITests(unittest.TestCase):
|
|
191
|
+
def run_cli(self, *args, **kwargs):
|
|
192
|
+
return subprocess.run([sys.executable, '-m', 'streaming_claim_audit', *map(str, args)], capture_output=True, text=True, **kwargs)
|
|
193
|
+
|
|
194
|
+
def test_good(self):
|
|
195
|
+
run = self.run_cli(FIXTURES / 'good')
|
|
196
|
+
self.assertEqual(run.returncode, 0)
|
|
197
|
+
self.assertEqual(run.stdout.strip(), 'PASS')
|
|
198
|
+
|
|
199
|
+
def test_bad_json(self):
|
|
200
|
+
run = self.run_cli(FIXTURES / 'bad', '--json')
|
|
201
|
+
self.assertEqual(run.returncode, 2)
|
|
202
|
+
result = json.loads(run.stdout)
|
|
203
|
+
self.assertEqual(set(result), {'findings', 'verified_claims', 'summary'})
|
|
204
|
+
self.assertEqual(result['summary']['high'], 2)
|
|
205
|
+
|
|
206
|
+
def test_testonly_default(self):
|
|
207
|
+
run = self.run_cli(FIXTURES / 'testonly')
|
|
208
|
+
self.assertEqual(run.returncode, 2)
|
|
209
|
+
self.assertEqual(run.stdout.count('MEDIUM '), 2)
|
|
210
|
+
|
|
211
|
+
def test_testonly_high(self):
|
|
212
|
+
run = self.run_cli(FIXTURES / 'testonly', '--fail-on', 'HIGH')
|
|
213
|
+
self.assertEqual(run.returncode, 0)
|
|
214
|
+
self.assertEqual(run.stdout.count('MEDIUM '), 2)
|
|
215
|
+
|
|
216
|
+
def test_version(self):
|
|
217
|
+
run = self.run_cli('--version')
|
|
218
|
+
self.assertEqual((run.returncode, run.stdout.strip()), (0, '0.1.0'))
|
|
219
|
+
|
|
220
|
+
def test_bad_path(self):
|
|
221
|
+
self.assertEqual(self.run_cli(FIXTURES / 'missing').returncode, 1)
|
|
222
|
+
|
|
223
|
+
def test_bad_option(self):
|
|
224
|
+
self.assertEqual(self.run_cli('--unknown').returncode, 1)
|
|
225
|
+
|
|
226
|
+
def test_bad_threshold(self):
|
|
227
|
+
self.assertEqual(self.run_cli('--fail-on', 'LOW').returncode, 1)
|
|
228
|
+
|
|
229
|
+
def test_ignore_cli(self):
|
|
230
|
+
run = self.run_cli(FIXTURES / 'bad', '--ignore', ','.join(RULES.values()))
|
|
231
|
+
self.assertEqual((run.returncode, run.stdout.strip()), (0, 'PASS'))
|
|
232
|
+
|
|
233
|
+
def test_verbose(self):
|
|
234
|
+
self.assertIn('VERIFIED STREAMING', self.run_cli(FIXTURES / 'good', '--verbose').stdout)
|
|
235
|
+
|
|
236
|
+
def test_json_verified(self):
|
|
237
|
+
self.assertEqual(len(json.loads(self.run_cli(FIXTURES / 'good', '--json').stdout)['verified_claims']), 1)
|
|
238
|
+
|
|
239
|
+
def test_default_path(self):
|
|
240
|
+
env = dict(os.environ, PYTHONPATH=str(Path(__file__).resolve().parents[1] / 'src'))
|
|
241
|
+
self.assertEqual(self.run_cli(cwd=FIXTURES / 'good', env=env).returncode, 0)
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
if __name__ == '__main__':
|
|
245
|
+
unittest.main()
|