aicheck 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- aicheck-0.1.0/LICENSE +21 -0
- aicheck-0.1.0/PKG-INFO +105 -0
- aicheck-0.1.0/README.md +75 -0
- aicheck-0.1.0/pyproject.toml +69 -0
- aicheck-0.1.0/setup.cfg +4 -0
- aicheck-0.1.0/src/aicheck/__init__.py +3 -0
- aicheck-0.1.0/src/aicheck/__main__.py +4 -0
- aicheck-0.1.0/src/aicheck/analyzer.py +91 -0
- aicheck-0.1.0/src/aicheck/cli.py +53 -0
- aicheck-0.1.0/src/aicheck/detectors/__init__.py +0 -0
- aicheck-0.1.0/src/aicheck/detectors/api_usage.py +76 -0
- aicheck-0.1.0/src/aicheck/detectors/dead_code.py +65 -0
- aicheck-0.1.0/src/aicheck/detectors/hallucination.py +44 -0
- aicheck-0.1.0/src/aicheck/models.py +51 -0
- aicheck-0.1.0/src/aicheck/reporters/__init__.py +0 -0
- aicheck-0.1.0/src/aicheck/reporters/console.py +65 -0
- aicheck-0.1.0/src/aicheck.egg-info/PKG-INFO +105 -0
- aicheck-0.1.0/src/aicheck.egg-info/SOURCES.txt +21 -0
- aicheck-0.1.0/src/aicheck.egg-info/dependency_links.txt +1 -0
- aicheck-0.1.0/src/aicheck.egg-info/entry_points.txt +2 -0
- aicheck-0.1.0/src/aicheck.egg-info/requires.txt +6 -0
- aicheck-0.1.0/src/aicheck.egg-info/top_level.txt +1 -0
- aicheck-0.1.0/tests/test_analyzer.py +159 -0
aicheck-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Mahesh Makvana
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
aicheck-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: aicheck
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Catch AI-generated code issues before they catch you
|
|
5
|
+
Author-email: Mahesh Makvana <maheshmakvana@users.noreply.github.com>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/maheshmakvana/ai-check
|
|
8
|
+
Project-URL: Repository, https://github.com/maheshmakvana/ai-check
|
|
9
|
+
Project-URL: BugTracker, https://github.com/maheshmakvana/ai-check/issues
|
|
10
|
+
Keywords: ai verification,code review,hallucination detection,llm code,ai code quality,code analysis,static analysis,python
|
|
11
|
+
Classifier: Development Status :: 3 - Alpha
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: Operating System :: OS Independent
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
19
|
+
Classifier: Topic :: Software Development :: Quality Assurance
|
|
20
|
+
Classifier: Topic :: Software Development :: Testing
|
|
21
|
+
Requires-Python: >=3.10
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
License-File: LICENSE
|
|
24
|
+
Requires-Dist: requests>=2.31.0
|
|
25
|
+
Provides-Extra: test
|
|
26
|
+
Requires-Dist: pytest>=7.0; extra == "test"
|
|
27
|
+
Requires-Dist: ruff>=0.3.0; extra == "test"
|
|
28
|
+
Requires-Dist: mypy>=1.8.0; extra == "test"
|
|
29
|
+
Dynamic: license-file
|
|
30
|
+
|
|
31
|
+
# aicheck
|
|
32
|
+
|
|
33
|
+
**Catch AI-generated code issues before they catch you.**
|
|
34
|
+
|
|
35
|
+
`aicheck` is a static analysis toolkit that detects common failure patterns in AI-generated Python code:
|
|
36
|
+
|
|
37
|
+
- **Hallucinated imports** — modules LLMs frequently invent (e.g. `utils`, `helpers`, `misc`)
|
|
38
|
+
- **Dead code** — unused variables, unreachable branches, code after `return`/`raise`
|
|
39
|
+
- **Suspicious API usage** — wrong method names for stdlib modules, `open().read()` patterns
|
|
40
|
+
- **Confidence scoring** — each file gets a 0–100 score based on findings severity
|
|
41
|
+
|
|
42
|
+
## Installation
|
|
43
|
+
|
|
44
|
+
```bash
|
|
45
|
+
pip install aicheck
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
## Quick Start
|
|
49
|
+
|
|
50
|
+
```bash
|
|
51
|
+
# Check a single file
|
|
52
|
+
aicheck check my_file.py
|
|
53
|
+
|
|
54
|
+
# Check an entire project
|
|
55
|
+
aicheck check src/
|
|
56
|
+
|
|
57
|
+
# JSON output for CI integration
|
|
58
|
+
aicheck check src/ --format json
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
## Sample Output
|
|
62
|
+
|
|
63
|
+
```
|
|
64
|
+
[FAIL] src/suspicious.py (score: 62.0)
|
|
65
|
+
high L1:0 [hallucinated_import] Potentially hallucinated module: 'utils'
|
|
66
|
+
↳ Verify 'utils' exists; check PyPI or project dependencies
|
|
67
|
+
medium L14:4 [unreachable_code] Unreachable branch: condition is always False
|
|
68
|
+
low L11:4 [dead_code] Possibly unused variable: 'unused_var'
|
|
69
|
+
|
|
70
|
+
Files: 1 Passed: 0 Failed: 1
|
|
71
|
+
Average confidence score: 62.0/100
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
## CLI Reference
|
|
75
|
+
|
|
76
|
+
| Command | Description |
|
|
77
|
+
|---|---|
|
|
78
|
+
| `aicheck check <path>` | Analyze a file or directory |
|
|
79
|
+
| `aicheck check <path> --format json` | Output as JSON |
|
|
80
|
+
| `aicheck version` | Show version |
|
|
81
|
+
|
|
82
|
+
## Score Interpretation
|
|
83
|
+
|
|
84
|
+
| Score | Meaning |
|
|
85
|
+
|---|---|
|
|
86
|
+
| 100–90 | Clean |
|
|
87
|
+
| 89–70 | Minor issues |
|
|
88
|
+
| 69–50 | Moderate issues — review recommended |
|
|
89
|
+
| <50 | Critical — do not commit without review |
|
|
90
|
+
|
|
91
|
+
## Development
|
|
92
|
+
|
|
93
|
+
```bash
|
|
94
|
+
git clone https://github.com/maheshmakvana/ai-check.git
|
|
95
|
+
cd ai-check
|
|
96
|
+
python -m venv venv && source venv/bin/activate
|
|
97
|
+
pip install -e ".[test]"
|
|
98
|
+
pytest tests/ -v
|
|
99
|
+
ruff check src/
|
|
100
|
+
mypy src/
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
## License
|
|
104
|
+
|
|
105
|
+
MIT
|
aicheck-0.1.0/README.md
ADDED
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
# aicheck
|
|
2
|
+
|
|
3
|
+
**Catch AI-generated code issues before they catch you.**
|
|
4
|
+
|
|
5
|
+
`aicheck` is a static analysis toolkit that detects common failure patterns in AI-generated Python code:
|
|
6
|
+
|
|
7
|
+
- **Hallucinated imports** — modules LLMs frequently invent (e.g. `utils`, `helpers`, `misc`)
|
|
8
|
+
- **Dead code** — unused variables, unreachable branches, code after `return`/`raise`
|
|
9
|
+
- **Suspicious API usage** — wrong method names for stdlib modules, `open().read()` patterns
|
|
10
|
+
- **Confidence scoring** — each file gets a 0–100 score based on findings severity
|
|
11
|
+
|
|
12
|
+
## Installation
|
|
13
|
+
|
|
14
|
+
```bash
|
|
15
|
+
pip install aicheck
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
## Quick Start
|
|
19
|
+
|
|
20
|
+
```bash
|
|
21
|
+
# Check a single file
|
|
22
|
+
aicheck check my_file.py
|
|
23
|
+
|
|
24
|
+
# Check an entire project
|
|
25
|
+
aicheck check src/
|
|
26
|
+
|
|
27
|
+
# JSON output for CI integration
|
|
28
|
+
aicheck check src/ --format json
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
## Sample Output
|
|
32
|
+
|
|
33
|
+
```
|
|
34
|
+
[FAIL] src/suspicious.py (score: 62.0)
|
|
35
|
+
high L1:0 [hallucinated_import] Potentially hallucinated module: 'utils'
|
|
36
|
+
↳ Verify 'utils' exists; check PyPI or project dependencies
|
|
37
|
+
medium L14:4 [unreachable_code] Unreachable branch: condition is always False
|
|
38
|
+
low L11:4 [dead_code] Possibly unused variable: 'unused_var'
|
|
39
|
+
|
|
40
|
+
Files: 1 Passed: 0 Failed: 1
|
|
41
|
+
Average confidence score: 62.0/100
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
## CLI Reference
|
|
45
|
+
|
|
46
|
+
| Command | Description |
|
|
47
|
+
|---|---|
|
|
48
|
+
| `aicheck check <path>` | Analyze a file or directory |
|
|
49
|
+
| `aicheck check <path> --format json` | Output as JSON |
|
|
50
|
+
| `aicheck version` | Show version |
|
|
51
|
+
|
|
52
|
+
## Score Interpretation
|
|
53
|
+
|
|
54
|
+
| Score | Meaning |
|
|
55
|
+
|---|---|
|
|
56
|
+
| 100–90 | Clean |
|
|
57
|
+
| 89–70 | Minor issues |
|
|
58
|
+
| 69–50 | Moderate issues — review recommended |
|
|
59
|
+
| <50 | Critical — do not commit without review |
|
|
60
|
+
|
|
61
|
+
## Development
|
|
62
|
+
|
|
63
|
+
```bash
|
|
64
|
+
git clone https://github.com/maheshmakvana/ai-check.git
|
|
65
|
+
cd ai-check
|
|
66
|
+
python -m venv venv && source venv/bin/activate
|
|
67
|
+
pip install -e ".[test]"
|
|
68
|
+
pytest tests/ -v
|
|
69
|
+
ruff check src/
|
|
70
|
+
mypy src/
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
## License
|
|
74
|
+
|
|
75
|
+
MIT
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=68.0"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "aicheck"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Catch AI-generated code issues before they catch you"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = "MIT"
|
|
11
|
+
requires-python = ">=3.10"
|
|
12
|
+
authors = [
|
|
13
|
+
{ name = "Mahesh Makvana", email = "maheshmakvana@users.noreply.github.com" },
|
|
14
|
+
]
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Development Status :: 3 - Alpha",
|
|
17
|
+
"Intended Audience :: Developers",
|
|
18
|
+
"Operating System :: OS Independent",
|
|
19
|
+
"Programming Language :: Python :: 3",
|
|
20
|
+
"Programming Language :: Python :: 3.10",
|
|
21
|
+
"Programming Language :: Python :: 3.11",
|
|
22
|
+
"Programming Language :: Python :: 3.12",
|
|
23
|
+
"Programming Language :: Python :: 3.13",
|
|
24
|
+
"Topic :: Software Development :: Quality Assurance",
|
|
25
|
+
"Topic :: Software Development :: Testing",
|
|
26
|
+
]
|
|
27
|
+
keywords = [
|
|
28
|
+
"ai verification", "code review", "hallucination detection",
|
|
29
|
+
"llm code", "ai code quality", "code analysis",
|
|
30
|
+
"static analysis", "python",
|
|
31
|
+
]
|
|
32
|
+
dependencies = [
|
|
33
|
+
"requests>=2.31.0",
|
|
34
|
+
]
|
|
35
|
+
|
|
36
|
+
[project.optional-dependencies]
|
|
37
|
+
test = [
|
|
38
|
+
"pytest>=7.0",
|
|
39
|
+
"ruff>=0.3.0",
|
|
40
|
+
"mypy>=1.8.0",
|
|
41
|
+
]
|
|
42
|
+
|
|
43
|
+
[project.urls]
|
|
44
|
+
Homepage = "https://github.com/maheshmakvana/ai-check"
|
|
45
|
+
Repository = "https://github.com/maheshmakvana/ai-check"
|
|
46
|
+
BugTracker = "https://github.com/maheshmakvana/ai-check/issues"
|
|
47
|
+
|
|
48
|
+
[project.scripts]
|
|
49
|
+
aicheck = "aicheck.cli:main"
|
|
50
|
+
|
|
51
|
+
[tool.setuptools.packages.find]
|
|
52
|
+
where = ["src"]
|
|
53
|
+
|
|
54
|
+
[tool.pytest.ini_options]
|
|
55
|
+
testpaths = ["tests"]
|
|
56
|
+
python_files = ["test_*.py"]
|
|
57
|
+
python_functions = ["test_*"]
|
|
58
|
+
|
|
59
|
+
[tool.ruff]
|
|
60
|
+
target-version = "py310"
|
|
61
|
+
line-length = 100
|
|
62
|
+
|
|
63
|
+
[tool.ruff.lint]
|
|
64
|
+
select = ["E", "F", "W", "I", "N", "UP", "S"]
|
|
65
|
+
|
|
66
|
+
[tool.mypy]
|
|
67
|
+
python_version = "3.10"
|
|
68
|
+
strict = true
|
|
69
|
+
ignore_missing_imports = true
|
aicheck-0.1.0/setup.cfg
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
import ast
|
|
2
|
+
import importlib
|
|
3
|
+
import sys
|
|
4
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import Protocol
|
|
7
|
+
|
|
8
|
+
import requests
|
|
9
|
+
|
|
10
|
+
from aicheck.detectors.api_usage import ApiUsageDetector
|
|
11
|
+
from aicheck.detectors.dead_code import DeadCodeDetector
|
|
12
|
+
from aicheck.detectors.hallucination import HallucinationDetector
|
|
13
|
+
from aicheck.models import FileResult, Finding
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class DetectorProtocol(Protocol):
|
|
17
|
+
def check(self, tree: ast.AST, source: str) -> list[Finding]:
|
|
18
|
+
...
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class Analyzer:
|
|
22
|
+
def __init__(self, timeout: int = 5):
|
|
23
|
+
self.timeout = timeout
|
|
24
|
+
self._detectors: list[DetectorProtocol] = [
|
|
25
|
+
HallucinationDetector(),
|
|
26
|
+
DeadCodeDetector(),
|
|
27
|
+
ApiUsageDetector(),
|
|
28
|
+
]
|
|
29
|
+
|
|
30
|
+
def check_file(self, path: Path) -> FileResult:
|
|
31
|
+
with path.open(encoding="utf-8", errors="replace") as f:
|
|
32
|
+
source = f.read()
|
|
33
|
+
try:
|
|
34
|
+
tree = ast.parse(source, filename=str(path))
|
|
35
|
+
except SyntaxError:
|
|
36
|
+
return FileResult(path=path)
|
|
37
|
+
result = FileResult(path=path)
|
|
38
|
+
for detector in self._detectors:
|
|
39
|
+
result.findings.extend(detector.check(tree, source))
|
|
40
|
+
return result
|
|
41
|
+
|
|
42
|
+
def check_path(self, path: Path) -> list[FileResult]:
|
|
43
|
+
if path.is_file() and path.suffix == ".py":
|
|
44
|
+
return [self.check_file(path)]
|
|
45
|
+
results: list[FileResult] = []
|
|
46
|
+
files = list(path.rglob("*.py"))
|
|
47
|
+
with ThreadPoolExecutor(max_workers=8) as pool:
|
|
48
|
+
for r in pool.map(self.check_file, files):
|
|
49
|
+
results.append(r)
|
|
50
|
+
return results
|
|
51
|
+
|
|
52
|
+
@staticmethod
|
|
53
|
+
def known_modules() -> set[str]:
|
|
54
|
+
known = set(sys.builtin_module_names)
|
|
55
|
+
for m in list(sys.modules.keys()):
|
|
56
|
+
parts = m.split(".")
|
|
57
|
+
for i in range(len(parts)):
|
|
58
|
+
known.add(".".join(parts[: i + 1]))
|
|
59
|
+
known.update({
|
|
60
|
+
"os", "sys", "re", "json", "math", "datetime", "pathlib",
|
|
61
|
+
"collections", "functools", "itertools", "typing", "abc",
|
|
62
|
+
"dataclasses", "enum", "hashlib", "random", "statistics",
|
|
63
|
+
"uuid", "inspect", "textwrap", "string", "decimal", "fractions",
|
|
64
|
+
"io", "base64", "binascii", "pickle", "shelve", "sqlite3",
|
|
65
|
+
"csv", "configparser", "logging", "argparse", "subprocess",
|
|
66
|
+
"shutil", "tempfile", "glob", "fnmatch", "linecache",
|
|
67
|
+
"asyncio", "threading", "multiprocessing", "concurrent",
|
|
68
|
+
"http", "urllib", "socket", "ssl", "email", "xml", "html",
|
|
69
|
+
"unittest", "doctest", "pdb", "profile", "timeit",
|
|
70
|
+
"warnings", "contextlib", "atexit", "weakref", "copy",
|
|
71
|
+
"pprint", "reprlib", "enum", "numbers", "stat",
|
|
72
|
+
"calendar", "locale", "gettext", "platform",
|
|
73
|
+
})
|
|
74
|
+
return known
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def verify_import(name: str, timeout: int = 3) -> bool:
|
|
78
|
+
top = name.split(".")[0]
|
|
79
|
+
try:
|
|
80
|
+
importlib.import_module(top)
|
|
81
|
+
return True
|
|
82
|
+
except ImportError:
|
|
83
|
+
pass
|
|
84
|
+
try:
|
|
85
|
+
resp = requests.get(
|
|
86
|
+
f"https://pypi.org/pypi/{top}/json",
|
|
87
|
+
timeout=timeout,
|
|
88
|
+
)
|
|
89
|
+
return bool(resp.status_code == 200)
|
|
90
|
+
except requests.RequestException:
|
|
91
|
+
return False
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import argparse
|
|
2
|
+
import sys
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
from aicheck.analyzer import Analyzer
|
|
6
|
+
from aicheck.reporters.console import get_reporter
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
10
|
+
parser = argparse.ArgumentParser(
|
|
11
|
+
prog="aicheck",
|
|
12
|
+
description="Catch AI-generated code issues before they catch you",
|
|
13
|
+
)
|
|
14
|
+
sub = parser.add_subparsers(dest="command")
|
|
15
|
+
|
|
16
|
+
check = sub.add_parser("check", help="Analyze Python files for AI code issues")
|
|
17
|
+
check.add_argument("path", type=str, nargs="?", default=".",
|
|
18
|
+
help="File or directory to check")
|
|
19
|
+
check.add_argument("--format", choices=["console", "json"], default="console",
|
|
20
|
+
help="Output format")
|
|
21
|
+
|
|
22
|
+
sub.add_parser("version", help="Show version")
|
|
23
|
+
return parser
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def main(argv: list[str] | None = None) -> int:
|
|
27
|
+
parser = build_parser()
|
|
28
|
+
args = parser.parse_args(argv)
|
|
29
|
+
|
|
30
|
+
if args.command == "version" or not args.command:
|
|
31
|
+
from aicheck import __version__
|
|
32
|
+
print(f"aicheck v{__version__}")
|
|
33
|
+
return 0
|
|
34
|
+
|
|
35
|
+
if args.command == "check":
|
|
36
|
+
target = Path(args.path).resolve()
|
|
37
|
+
if not target.exists():
|
|
38
|
+
print(f"Error: path '{target}' does not exist", file=sys.stderr)
|
|
39
|
+
return 1
|
|
40
|
+
analyzer = Analyzer()
|
|
41
|
+
results = analyzer.check_path(target)
|
|
42
|
+
reporter = get_reporter(args.format)
|
|
43
|
+
output = reporter.report(results)
|
|
44
|
+
print(output)
|
|
45
|
+
all_passed = all(r.passed for r in results)
|
|
46
|
+
return 0 if all_passed else 1
|
|
47
|
+
|
|
48
|
+
parser.print_help()
|
|
49
|
+
return 0
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
if __name__ == "__main__":
|
|
53
|
+
sys.exit(main())
|
|
File without changes
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
import ast
|
|
2
|
+
import sys
|
|
3
|
+
|
|
4
|
+
from aicheck.models import Finding, FindingKind, Severity
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class ApiUsageDetector:
|
|
8
|
+
FAKE_FUNCTIONS: dict[str, set[str]] = {
|
|
9
|
+
"os": {"path.join"},
|
|
10
|
+
"sys": {"argv"},
|
|
11
|
+
"re": {"search", "match", "findall", "sub", "split"},
|
|
12
|
+
"json": {"dumps", "loads", "dump", "load"},
|
|
13
|
+
"typing": {"List", "Dict", "Optional", "Tuple", "Union", "Any"},
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
REAL_STDLIB_FUNCTIONS: set[str] = set()
|
|
17
|
+
|
|
18
|
+
def __init__(self) -> None:
|
|
19
|
+
if not self.REAL_STDLIB_FUNCTIONS:
|
|
20
|
+
self._build_stdlib_index()
|
|
21
|
+
|
|
22
|
+
@staticmethod
|
|
23
|
+
def _build_stdlib_index() -> None:
|
|
24
|
+
known: set[str] = set()
|
|
25
|
+
for mod_name in sys.stdlib_module_names:
|
|
26
|
+
try:
|
|
27
|
+
mod = __import__(mod_name)
|
|
28
|
+
for attr in dir(mod):
|
|
29
|
+
known.add(f"{mod_name}.{attr}")
|
|
30
|
+
except (ImportError, AttributeError):
|
|
31
|
+
pass
|
|
32
|
+
ApiUsageDetector.REAL_STDLIB_FUNCTIONS = known
|
|
33
|
+
|
|
34
|
+
def check(self, tree: ast.AST, source: str) -> list[Finding]:
|
|
35
|
+
findings: list[Finding] = []
|
|
36
|
+
self._check_wrong_stdlib_methods(tree, findings)
|
|
37
|
+
self._check_common_mistakes(tree, findings)
|
|
38
|
+
return findings
|
|
39
|
+
|
|
40
|
+
@staticmethod
|
|
41
|
+
def _check_wrong_stdlib_methods(tree: ast.AST, findings: list[Finding]) -> None:
|
|
42
|
+
for node in ast.walk(tree):
|
|
43
|
+
if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute):
|
|
44
|
+
if isinstance(node.func.value, ast.Name):
|
|
45
|
+
mod = node.func.value.id
|
|
46
|
+
func = f"{mod}.{node.func.attr}"
|
|
47
|
+
if mod in ("os", "sys", "json", "re", "typing"):
|
|
48
|
+
expected = ApiUsageDetector.FAKE_FUNCTIONS.get(mod, set())
|
|
49
|
+
if node.func.attr not in expected:
|
|
50
|
+
findings.append(Finding(
|
|
51
|
+
kind=FindingKind.SUSPICIOUS_API,
|
|
52
|
+
message=f"Suspicious API call: '{func}' — "
|
|
53
|
+
f"uncommon for module '{mod}'",
|
|
54
|
+
line=node.lineno or 0,
|
|
55
|
+
column=node.col_offset or 0,
|
|
56
|
+
severity=Severity.MEDIUM,
|
|
57
|
+
suggestion=f"Verify '{func}' is a valid {mod} function",
|
|
58
|
+
))
|
|
59
|
+
|
|
60
|
+
@staticmethod
|
|
61
|
+
def _check_common_mistakes(tree: ast.AST, findings: list[Finding]) -> None:
|
|
62
|
+
for node in ast.walk(tree):
|
|
63
|
+
if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute):
|
|
64
|
+
if isinstance(node.func.value, ast.Name):
|
|
65
|
+
if node.func.value.id == "open" and node.func.attr == "read":
|
|
66
|
+
findings.append(Finding(
|
|
67
|
+
kind=FindingKind.SUSPICIOUS_API,
|
|
68
|
+
message=(
|
|
69
|
+
"Suspicious: 'open(...).read()' — "
|
|
70
|
+
"use 'pathlib.Path.read_text()' instead"
|
|
71
|
+
),
|
|
72
|
+
line=node.lineno or 0,
|
|
73
|
+
column=node.col_offset or 0,
|
|
74
|
+
severity=Severity.LOW,
|
|
75
|
+
suggestion="Replace with Path.read_text()",
|
|
76
|
+
))
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
import ast
|
|
2
|
+
|
|
3
|
+
from aicheck.models import Finding, FindingKind, Severity
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class DeadCodeDetector:
|
|
7
|
+
def check(self, tree: ast.AST, source: str) -> list[Finding]:
|
|
8
|
+
findings: list[Finding] = []
|
|
9
|
+
self._check_unreachable(tree, findings)
|
|
10
|
+
self._check_unused_vars(tree, findings)
|
|
11
|
+
self._check_dead_after_return(tree, findings)
|
|
12
|
+
return findings
|
|
13
|
+
|
|
14
|
+
@staticmethod
|
|
15
|
+
def _check_unreachable(tree: ast.AST, findings: list[Finding]) -> None:
|
|
16
|
+
for node in ast.walk(tree):
|
|
17
|
+
if isinstance(node, ast.If):
|
|
18
|
+
if isinstance(node.test, ast.Constant) and node.test.value is False:
|
|
19
|
+
findings.append(Finding(
|
|
20
|
+
kind=FindingKind.UNREACHABLE_CODE,
|
|
21
|
+
message="Unreachable branch: condition is always False",
|
|
22
|
+
line=node.lineno or 0,
|
|
23
|
+
column=node.col_offset or 0,
|
|
24
|
+
severity=Severity.MEDIUM,
|
|
25
|
+
suggestion="Remove dead branch or update condition",
|
|
26
|
+
))
|
|
27
|
+
|
|
28
|
+
@staticmethod
|
|
29
|
+
def _check_unused_vars(tree: ast.AST, findings: list[Finding]) -> None:
|
|
30
|
+
assigns: dict[str, ast.Name] = {}
|
|
31
|
+
for node in ast.walk(tree):
|
|
32
|
+
if isinstance(node, ast.Assign):
|
|
33
|
+
for t in node.targets:
|
|
34
|
+
if isinstance(t, ast.Name) and not t.id.startswith("_"):
|
|
35
|
+
assigns[t.id] = t
|
|
36
|
+
for var, node in assigns.items():
|
|
37
|
+
count = sum(1 for n in ast.walk(tree)
|
|
38
|
+
if isinstance(n, ast.Name) and n.id == var)
|
|
39
|
+
if count <= 1:
|
|
40
|
+
findings.append(Finding(
|
|
41
|
+
kind=FindingKind.DEAD_CODE,
|
|
42
|
+
message=f"Possibly unused variable: '{var}' (assigned but never read)",
|
|
43
|
+
line=node.lineno or 0,
|
|
44
|
+
column=node.col_offset or 0,
|
|
45
|
+
severity=Severity.LOW,
|
|
46
|
+
suggestion=f"Remove '{var}' or use it elsewhere",
|
|
47
|
+
))
|
|
48
|
+
|
|
49
|
+
@staticmethod
|
|
50
|
+
def _check_dead_after_return(tree: ast.AST, findings: list[Finding]) -> None:
|
|
51
|
+
for node in ast.walk(tree):
|
|
52
|
+
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
|
53
|
+
stmts = node.body
|
|
54
|
+
for i, stmt in enumerate(stmts[:-1]):
|
|
55
|
+
if isinstance(stmt, (ast.Return, ast.Raise)):
|
|
56
|
+
next_stmt = stmts[i + 1]
|
|
57
|
+
findings.append(Finding(
|
|
58
|
+
kind=FindingKind.DEAD_CODE,
|
|
59
|
+
message=f"Dead code after {type(stmt).__name__.lower()} "
|
|
60
|
+
f"on line {next_stmt.lineno}",
|
|
61
|
+
line=next_stmt.lineno or 0,
|
|
62
|
+
column=next_stmt.col_offset or 0,
|
|
63
|
+
severity=Severity.MEDIUM,
|
|
64
|
+
suggestion="Remove unreachable statement or reorder logic",
|
|
65
|
+
))
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import ast
|
|
2
|
+
|
|
3
|
+
from aicheck.models import Finding, FindingKind, Severity
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class HallucinationDetector:
|
|
7
|
+
FAKE_STDLIB_MODULES: set[str] = {
|
|
8
|
+
"utils", "helpers", "common", "tools", "misc",
|
|
9
|
+
"extras", "general", "core", "base", "utilities",
|
|
10
|
+
"datatools", "fileutils", "strutils", "validators",
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
def check(self, tree: ast.AST, source: str) -> list[Finding]:
|
|
14
|
+
findings: list[Finding] = []
|
|
15
|
+
for node in ast.walk(tree):
|
|
16
|
+
if isinstance(node, ast.Import):
|
|
17
|
+
for alias in node.names:
|
|
18
|
+
top = alias.name.split(".")[0]
|
|
19
|
+
if top in self.FAKE_STDLIB_MODULES:
|
|
20
|
+
findings.append(Finding(
|
|
21
|
+
kind=FindingKind.HALLUCINATED_IMPORT,
|
|
22
|
+
message=f"Potentially hallucinated module: '{alias.name}' — "
|
|
23
|
+
f"'{top}' is a common LLM-invented name",
|
|
24
|
+
line=node.lineno or 0,
|
|
25
|
+
column=node.col_offset or 0,
|
|
26
|
+
severity=Severity.HIGH,
|
|
27
|
+
suggestion=(
|
|
28
|
+
f"Verify '{alias.name}' exists; "
|
|
29
|
+
"check PyPI or project dependencies"
|
|
30
|
+
),
|
|
31
|
+
))
|
|
32
|
+
elif isinstance(node, ast.ImportFrom):
|
|
33
|
+
if node.module:
|
|
34
|
+
top = node.module.split(".")[0]
|
|
35
|
+
if top in self.FAKE_STDLIB_MODULES and node.level is None:
|
|
36
|
+
findings.append(Finding(
|
|
37
|
+
kind=FindingKind.FAKE_STDLIB,
|
|
38
|
+
message=f"Import from potentially fake module: '{node.module}'",
|
|
39
|
+
line=node.lineno or 0,
|
|
40
|
+
column=node.col_offset or 0,
|
|
41
|
+
severity=Severity.HIGH,
|
|
42
|
+
suggestion=f"Ensure '{node.module}' is a real package",
|
|
43
|
+
))
|
|
44
|
+
return findings
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
from dataclasses import dataclass, field
|
|
2
|
+
from enum import Enum
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class Severity(Enum):
|
|
7
|
+
LOW = "low"
|
|
8
|
+
MEDIUM = "medium"
|
|
9
|
+
HIGH = "high"
|
|
10
|
+
CRITICAL = "critical"
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class FindingKind(Enum):
|
|
14
|
+
HALLUCINATED_IMPORT = "hallucinated_import"
|
|
15
|
+
DEAD_CODE = "dead_code"
|
|
16
|
+
SUSPICIOUS_API = "suspicious_api"
|
|
17
|
+
UNREACHABLE_CODE = "unreachable_code"
|
|
18
|
+
FAKE_STDLIB = "fake_stdlib"
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@dataclass
|
|
22
|
+
class Finding:
|
|
23
|
+
kind: FindingKind
|
|
24
|
+
message: str
|
|
25
|
+
line: int
|
|
26
|
+
column: int
|
|
27
|
+
severity: Severity
|
|
28
|
+
suggestion: str = ""
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@dataclass
|
|
32
|
+
class FileResult:
|
|
33
|
+
path: Path
|
|
34
|
+
findings: list[Finding] = field(default_factory=list)
|
|
35
|
+
|
|
36
|
+
@property
|
|
37
|
+
def score(self) -> float:
|
|
38
|
+
if not self.findings:
|
|
39
|
+
return 100.0
|
|
40
|
+
penalties = {
|
|
41
|
+
Severity.LOW: 2,
|
|
42
|
+
Severity.MEDIUM: 5,
|
|
43
|
+
Severity.HIGH: 15,
|
|
44
|
+
Severity.CRITICAL: 30,
|
|
45
|
+
}
|
|
46
|
+
total_penalty = sum(penalties[f.severity] for f in self.findings)
|
|
47
|
+
return max(0.0, 100.0 - total_penalty)
|
|
48
|
+
|
|
49
|
+
@property
|
|
50
|
+
def passed(self) -> bool:
|
|
51
|
+
return self.score >= 70.0
|
|
File without changes
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
import json
|
|
2
|
+
|
|
3
|
+
from aicheck.models import FileResult
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class ConsoleReporter:
|
|
7
|
+
def report(self, results: list[FileResult]) -> str:
|
|
8
|
+
lines: list[str] = []
|
|
9
|
+
passed = 0
|
|
10
|
+
failed = 0
|
|
11
|
+
for r in results:
|
|
12
|
+
status = "PASS" if r.passed else "FAIL"
|
|
13
|
+
if r.passed:
|
|
14
|
+
passed += 1
|
|
15
|
+
else:
|
|
16
|
+
failed += 1
|
|
17
|
+
lines.append(f"[{status}] {r.path} (score: {r.score:.1f})")
|
|
18
|
+
for f in r.findings:
|
|
19
|
+
lines.append(
|
|
20
|
+
f" {f.severity.value:>8} L{f.line}:{f.column} "
|
|
21
|
+
f"[{f.kind.value}] {f.message}"
|
|
22
|
+
)
|
|
23
|
+
if f.suggestion:
|
|
24
|
+
lines.append(f" \u21b3 {f.suggestion}")
|
|
25
|
+
if not results:
|
|
26
|
+
lines.append("No Python files found.")
|
|
27
|
+
lines.append("")
|
|
28
|
+
total = len(results)
|
|
29
|
+
lines.append(f"Files: {total} Passed: {passed} Failed: {failed}")
|
|
30
|
+
if total:
|
|
31
|
+
avg = sum(r.score for r in results) / total
|
|
32
|
+
lines.append(f"Average confidence score: {avg:.1f}/100")
|
|
33
|
+
return "\n".join(lines)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class JsonReporter:
|
|
37
|
+
def report(self, results: list[FileResult]) -> str:
|
|
38
|
+
data = []
|
|
39
|
+
for r in results:
|
|
40
|
+
data.append({
|
|
41
|
+
"path": str(r.path),
|
|
42
|
+
"score": r.score,
|
|
43
|
+
"passed": r.passed,
|
|
44
|
+
"findings": [
|
|
45
|
+
{
|
|
46
|
+
"kind": f.kind.value,
|
|
47
|
+
"severity": f.severity.value,
|
|
48
|
+
"line": f.line,
|
|
49
|
+
"column": f.column,
|
|
50
|
+
"message": f.message,
|
|
51
|
+
"suggestion": f.suggestion,
|
|
52
|
+
}
|
|
53
|
+
for f in r.findings
|
|
54
|
+
],
|
|
55
|
+
})
|
|
56
|
+
return json.dumps({"files": data}, indent=2)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def get_reporter(format: str) -> ConsoleReporter | JsonReporter:
|
|
60
|
+
reporters: dict[str, type[ConsoleReporter] | type[JsonReporter]] = {
|
|
61
|
+
"console": ConsoleReporter,
|
|
62
|
+
"json": JsonReporter,
|
|
63
|
+
}
|
|
64
|
+
cls = reporters.get(format, ConsoleReporter)
|
|
65
|
+
return cls()
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: aicheck
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Catch AI-generated code issues before they catch you
|
|
5
|
+
Author-email: Mahesh Makvana <maheshmakvana@users.noreply.github.com>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/maheshmakvana/ai-check
|
|
8
|
+
Project-URL: Repository, https://github.com/maheshmakvana/ai-check
|
|
9
|
+
Project-URL: BugTracker, https://github.com/maheshmakvana/ai-check/issues
|
|
10
|
+
Keywords: ai verification,code review,hallucination detection,llm code,ai code quality,code analysis,static analysis,python
|
|
11
|
+
Classifier: Development Status :: 3 - Alpha
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: Operating System :: OS Independent
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
19
|
+
Classifier: Topic :: Software Development :: Quality Assurance
|
|
20
|
+
Classifier: Topic :: Software Development :: Testing
|
|
21
|
+
Requires-Python: >=3.10
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
License-File: LICENSE
|
|
24
|
+
Requires-Dist: requests>=2.31.0
|
|
25
|
+
Provides-Extra: test
|
|
26
|
+
Requires-Dist: pytest>=7.0; extra == "test"
|
|
27
|
+
Requires-Dist: ruff>=0.3.0; extra == "test"
|
|
28
|
+
Requires-Dist: mypy>=1.8.0; extra == "test"
|
|
29
|
+
Dynamic: license-file
|
|
30
|
+
|
|
31
|
+
# aicheck
|
|
32
|
+
|
|
33
|
+
**Catch AI-generated code issues before they catch you.**
|
|
34
|
+
|
|
35
|
+
`aicheck` is a static analysis toolkit that detects common failure patterns in AI-generated Python code:
|
|
36
|
+
|
|
37
|
+
- **Hallucinated imports** — modules LLMs frequently invent (e.g. `utils`, `helpers`, `misc`)
|
|
38
|
+
- **Dead code** — unused variables, unreachable branches, code after `return`/`raise`
|
|
39
|
+
- **Suspicious API usage** — wrong method names for stdlib modules, `open().read()` patterns
|
|
40
|
+
- **Confidence scoring** — each file gets a 0–100 score based on findings severity
|
|
41
|
+
|
|
42
|
+
## Installation
|
|
43
|
+
|
|
44
|
+
```bash
|
|
45
|
+
pip install aicheck
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
## Quick Start
|
|
49
|
+
|
|
50
|
+
```bash
|
|
51
|
+
# Check a single file
|
|
52
|
+
aicheck check my_file.py
|
|
53
|
+
|
|
54
|
+
# Check an entire project
|
|
55
|
+
aicheck check src/
|
|
56
|
+
|
|
57
|
+
# JSON output for CI integration
|
|
58
|
+
aicheck check src/ --format json
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
## Sample Output
|
|
62
|
+
|
|
63
|
+
```
|
|
64
|
+
[FAIL] src/suspicious.py (score: 62.0)
|
|
65
|
+
high L1:0 [hallucinated_import] Potentially hallucinated module: 'utils'
|
|
66
|
+
↳ Verify 'utils' exists; check PyPI or project dependencies
|
|
67
|
+
medium L14:4 [unreachable_code] Unreachable branch: condition is always False
|
|
68
|
+
low L11:4 [dead_code] Possibly unused variable: 'unused_var'
|
|
69
|
+
|
|
70
|
+
Files: 1 Passed: 0 Failed: 1
|
|
71
|
+
Average confidence score: 62.0/100
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
## CLI Reference
|
|
75
|
+
|
|
76
|
+
| Command | Description |
|
|
77
|
+
|---|---|
|
|
78
|
+
| `aicheck check <path>` | Analyze a file or directory |
|
|
79
|
+
| `aicheck check <path> --format json` | Output as JSON |
|
|
80
|
+
| `aicheck version` | Show version |
|
|
81
|
+
|
|
82
|
+
## Score Interpretation
|
|
83
|
+
|
|
84
|
+
| Score | Meaning |
|
|
85
|
+
|---|---|
|
|
86
|
+
| 100–90 | Clean |
|
|
87
|
+
| 89–70 | Minor issues |
|
|
88
|
+
| 69–50 | Moderate issues — review recommended |
|
|
89
|
+
| <50 | Critical — do not commit without review |
|
|
90
|
+
|
|
91
|
+
## Development
|
|
92
|
+
|
|
93
|
+
```bash
|
|
94
|
+
git clone https://github.com/maheshmakvana/ai-check.git
|
|
95
|
+
cd ai-check
|
|
96
|
+
python -m venv venv && source venv/bin/activate
|
|
97
|
+
pip install -e ".[test]"
|
|
98
|
+
pytest tests/ -v
|
|
99
|
+
ruff check src/
|
|
100
|
+
mypy src/
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
## License
|
|
104
|
+
|
|
105
|
+
MIT
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
README.md
|
|
3
|
+
pyproject.toml
|
|
4
|
+
src/aicheck/__init__.py
|
|
5
|
+
src/aicheck/__main__.py
|
|
6
|
+
src/aicheck/analyzer.py
|
|
7
|
+
src/aicheck/cli.py
|
|
8
|
+
src/aicheck/models.py
|
|
9
|
+
src/aicheck.egg-info/PKG-INFO
|
|
10
|
+
src/aicheck.egg-info/SOURCES.txt
|
|
11
|
+
src/aicheck.egg-info/dependency_links.txt
|
|
12
|
+
src/aicheck.egg-info/entry_points.txt
|
|
13
|
+
src/aicheck.egg-info/requires.txt
|
|
14
|
+
src/aicheck.egg-info/top_level.txt
|
|
15
|
+
src/aicheck/detectors/__init__.py
|
|
16
|
+
src/aicheck/detectors/api_usage.py
|
|
17
|
+
src/aicheck/detectors/dead_code.py
|
|
18
|
+
src/aicheck/detectors/hallucination.py
|
|
19
|
+
src/aicheck/reporters/__init__.py
|
|
20
|
+
src/aicheck/reporters/console.py
|
|
21
|
+
tests/test_analyzer.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
aicheck
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
import ast
|
|
2
|
+
import json
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
import pytest
|
|
6
|
+
|
|
7
|
+
from aicheck.analyzer import Analyzer
|
|
8
|
+
|
|
9
|
+
SAMPLES = Path(__file__).parent / "samples"
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@pytest.fixture
|
|
13
|
+
def analyzer() -> Analyzer:
|
|
14
|
+
return Analyzer()
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def test_check_good_code(analyzer: Analyzer) -> None:
|
|
18
|
+
result = analyzer.check_file(SAMPLES / "good_code.py")
|
|
19
|
+
assert result.score == 100.0
|
|
20
|
+
assert len(result.findings) == 0
|
|
21
|
+
assert result.passed
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def test_check_suspicious_code(analyzer: Analyzer) -> None:
|
|
25
|
+
result = analyzer.check_file(SAMPLES / "suspicious_code.py")
|
|
26
|
+
assert len(result.findings) >= 3
|
|
27
|
+
assert not result.passed
|
|
28
|
+
assert result.score < 100.0
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def test_check_directory(analyzer: Analyzer) -> None:
|
|
32
|
+
results = analyzer.check_path(SAMPLES)
|
|
33
|
+
assert len(results) == 2
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def test_unreachable_code_detection(analyzer: Analyzer) -> None:
|
|
37
|
+
source = """
|
|
38
|
+
if False:
|
|
39
|
+
print("never")
|
|
40
|
+
"""
|
|
41
|
+
tree = ast.parse(source)
|
|
42
|
+
all_findings = []
|
|
43
|
+
for d in analyzer._detectors:
|
|
44
|
+
all_findings.extend(d.check(tree, source))
|
|
45
|
+
unreachable = [f for f in all_findings if f.kind.name == "UNREACHABLE_CODE"]
|
|
46
|
+
assert len(unreachable) >= 1
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def test_unused_var_detection(analyzer: Analyzer) -> None:
|
|
50
|
+
source = """
|
|
51
|
+
x = 42
|
|
52
|
+
y = 10
|
|
53
|
+
print(y)
|
|
54
|
+
"""
|
|
55
|
+
tree = ast.parse(source)
|
|
56
|
+
all_findings = []
|
|
57
|
+
for d in analyzer._detectors:
|
|
58
|
+
all_findings.extend(d.check(tree, source))
|
|
59
|
+
unused = [f for f in all_findings
|
|
60
|
+
if f.kind.name == "DEAD_CODE" and "unused" in f.message.lower()]
|
|
61
|
+
assert len(unused) >= 1
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def test_json_output_format(analyzer: Analyzer) -> None:
|
|
65
|
+
from aicheck.reporters.console import JsonReporter
|
|
66
|
+
results = analyzer.check_path(SAMPLES)
|
|
67
|
+
reporter = JsonReporter()
|
|
68
|
+
output = reporter.report(results)
|
|
69
|
+
parsed = json.loads(output)
|
|
70
|
+
assert "files" in parsed
|
|
71
|
+
assert len(parsed["files"]) == 2
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def test_pass_threshold(analyzer: Analyzer) -> None:
|
|
75
|
+
result = analyzer.check_file(SAMPLES / "good_code.py")
|
|
76
|
+
assert result.passed
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def test_fail_threshold(analyzer: Analyzer) -> None:
|
|
80
|
+
result = analyzer.check_file(SAMPLES / "suspicious_code.py")
|
|
81
|
+
assert not result.passed
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def test_dead_after_return(analyzer: Analyzer) -> None:
|
|
85
|
+
source = """
|
|
86
|
+
def foo():
|
|
87
|
+
return 1
|
|
88
|
+
x = 2
|
|
89
|
+
"""
|
|
90
|
+
tree = ast.parse(source)
|
|
91
|
+
all_findings = []
|
|
92
|
+
for d in analyzer._detectors:
|
|
93
|
+
all_findings.extend(d.check(tree, source))
|
|
94
|
+
dead_after = [f for f in all_findings
|
|
95
|
+
if f.kind.name == "DEAD_CODE" and "Dead code after" in f.message]
|
|
96
|
+
assert len(dead_after) >= 1
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def test_cli_check_file(capsys: pytest.CaptureFixture) -> None:
|
|
100
|
+
from aicheck.cli import main
|
|
101
|
+
rc = main(["check", str(SAMPLES / "good_code.py")])
|
|
102
|
+
captured = capsys.readouterr()
|
|
103
|
+
assert rc == 0
|
|
104
|
+
assert "PASS" in captured.out
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def test_cli_check_suspicious(capsys: pytest.CaptureFixture) -> None:
|
|
108
|
+
from aicheck.cli import main
|
|
109
|
+
rc = main(["check", str(SAMPLES / "suspicious_code.py")])
|
|
110
|
+
captured = capsys.readouterr()
|
|
111
|
+
assert rc == 1
|
|
112
|
+
assert "FAIL" in captured.out
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def test_cli_json_format(capsys: pytest.CaptureFixture) -> None:
|
|
116
|
+
from aicheck.cli import main
|
|
117
|
+
rc = main(["check", str(SAMPLES / "good_code.py"), "--format", "json"])
|
|
118
|
+
captured = capsys.readouterr()
|
|
119
|
+
assert rc == 0
|
|
120
|
+
parsed = json.loads(captured.out)
|
|
121
|
+
assert parsed["files"][0]["score"] == 100.0
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def test_cli_version(capsys: pytest.CaptureFixture) -> None:
|
|
125
|
+
from aicheck.cli import main
|
|
126
|
+
rc = main(["version"])
|
|
127
|
+
captured = capsys.readouterr()
|
|
128
|
+
assert rc == 0
|
|
129
|
+
assert "aicheck" in captured.out
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def test_hallucinated_import_detection(analyzer: Analyzer) -> None:
|
|
133
|
+
source = "import utils\nimport helpers\n"
|
|
134
|
+
tree = ast.parse(source)
|
|
135
|
+
all_findings = []
|
|
136
|
+
for d in analyzer._detectors:
|
|
137
|
+
all_findings.extend(d.check(tree, source))
|
|
138
|
+
hallucinated = [f for f in all_findings
|
|
139
|
+
if f.kind.name in ("HALLUCINATED_IMPORT", "FAKE_STDLIB")]
|
|
140
|
+
assert len(hallucinated) >= 2
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def test_score_penalties() -> None:
|
|
144
|
+
from aicheck.models import FileResult, Finding, FindingKind, Severity
|
|
145
|
+
path = Path("dummy.py")
|
|
146
|
+
r = FileResult(path=path)
|
|
147
|
+
r.findings.append(Finding(
|
|
148
|
+
kind=FindingKind.DEAD_CODE, message="", line=1, column=0,
|
|
149
|
+
severity=Severity.CRITICAL,
|
|
150
|
+
))
|
|
151
|
+
assert r.score == 70.0
|
|
152
|
+
assert r.passed
|
|
153
|
+
|
|
154
|
+
r.findings.append(Finding(
|
|
155
|
+
kind=FindingKind.DEAD_CODE, message="", line=2, column=0,
|
|
156
|
+
severity=Severity.CRITICAL,
|
|
157
|
+
))
|
|
158
|
+
assert r.score == 40.0
|
|
159
|
+
assert not r.passed
|