skeptic-cli 0.5.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- skeptic/__init__.py +0 -0
- skeptic/adapters/__init__.py +0 -0
- skeptic/adapters/python/__init__.py +27 -0
- skeptic/adapters/python/bandit_adapter.py +61 -0
- skeptic/adapters/python/pip_audit_adapter.py +56 -0
- skeptic/adapters/python/pyright_adapter.py +45 -0
- skeptic/adapters/python/pytest_adapter.py +59 -0
- skeptic/adapters/python/ruff_adapter.py +44 -0
- skeptic/cli/__init__.py +0 -0
- skeptic/cli/main.py +337 -0
- skeptic/core/__init__.py +0 -0
- skeptic/core/complexity.py +110 -0
- skeptic/core/config.py +72 -0
- skeptic/core/gate.py +124 -0
- skeptic/core/git_utils.py +97 -0
- skeptic/core/graph.py +157 -0
- skeptic/core/models.py +69 -0
- skeptic/core/provenance.py +90 -0
- skeptic/core/risk_score.py +99 -0
- skeptic/core/runner.py +43 -0
- skeptic/core/solid.py +266 -0
- skeptic/mcp/__init__.py +0 -0
- skeptic/mcp/call_log.py +68 -0
- skeptic/mcp/server.py +101 -0
- skeptic/narration/__init__.py +3 -0
- skeptic/narration/gemini_narrator.py +92 -0
- skeptic/verifier/__init__.py +14 -0
- skeptic/verifier/attack_runner.py +144 -0
- skeptic/verifier/gemini_generator.py +93 -0
- skeptic/verifier/models.py +88 -0
- skeptic_cli-0.5.0.dist-info/METADATA +389 -0
- skeptic_cli-0.5.0.dist-info/RECORD +36 -0
- skeptic_cli-0.5.0.dist-info/WHEEL +5 -0
- skeptic_cli-0.5.0.dist-info/entry_points.txt +3 -0
- skeptic_cli-0.5.0.dist-info/licenses/LICENSE +21 -0
- skeptic_cli-0.5.0.dist-info/top_level.txt +1 -0
skeptic/__init__.py
ADDED
|
File without changes
|
|
File without changes
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
from skeptic.adapters.python.bandit_adapter import run_bandit
|
|
2
|
+
from skeptic.adapters.python.pip_audit_adapter import run_pip_audit
|
|
3
|
+
from skeptic.adapters.python.pyright_adapter import run_pyright
|
|
4
|
+
from skeptic.adapters.python.pytest_adapter import run_pytest
|
|
5
|
+
from skeptic.adapters.python.ruff_adapter import run_ruff
|
|
6
|
+
from skeptic.core.complexity import check_complexity
|
|
7
|
+
from skeptic.core.solid import run_solid_checks
|
|
8
|
+
|
|
9
|
+
FINDING_ADAPTERS = {
|
|
10
|
+
"ruff": run_ruff,
|
|
11
|
+
"pyright": run_pyright,
|
|
12
|
+
"bandit": run_bandit,
|
|
13
|
+
"pip-audit": run_pip_audit,
|
|
14
|
+
# Phase 3 milestone 2: structural SOLID checks. Not tool-specific (no
|
|
15
|
+
# external binary), lives in core/ - registered here purely so
|
|
16
|
+
# run_all_checks picks it up the same way as everything else. Findings
|
|
17
|
+
# show up in `skeptic check` output/--json but don't fail the gate yet -
|
|
18
|
+
# gate.py has no rule for tool="solid" until Phase 3 milestone 8 wires
|
|
19
|
+
# up architecture/ai_review thresholds in skeptic.yaml.
|
|
20
|
+
"solid": run_solid_checks,
|
|
21
|
+
# Phase 3 milestone 6 plumbing: McCabe complexity, feeding the Change
|
|
22
|
+
# Risk Score's "complexity" dimension. Same non-gated-yet status as
|
|
23
|
+
# "solid" above.
|
|
24
|
+
"complexity": check_complexity,
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
TEST_ADAPTER = ("pytest", run_pytest)
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
"""Wraps `bandit -r -f json` into the shared Finding schema."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import json
|
|
5
|
+
import subprocess
|
|
6
|
+
|
|
7
|
+
from skeptic.core.models import Finding
|
|
8
|
+
|
|
9
|
+
_SEVERITY_MAP = {
|
|
10
|
+
"HIGH": "critical",
|
|
11
|
+
"MEDIUM": "high",
|
|
12
|
+
"LOW": "medium",
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def run_bandit(repo_path: str) -> list[Finding]:
|
|
17
|
+
proc = subprocess.run(
|
|
18
|
+
[
|
|
19
|
+
"bandit",
|
|
20
|
+
"-r",
|
|
21
|
+
repo_path,
|
|
22
|
+
"-f",
|
|
23
|
+
"json",
|
|
24
|
+
# Test code legitimately uses `assert` (bandit B101) as part of
|
|
25
|
+
# normal test authoring, not as a bypassable security control,
|
|
26
|
+
# and non-cryptographic `random` (B311) for fixture/test data.
|
|
27
|
+
# Scanning it produces noise, not signal. Cover both the common
|
|
28
|
+
# filename conventions (test_*.py / *_test.py) and files that
|
|
29
|
+
# live under a tests/ directory without matching those names
|
|
30
|
+
# (e.g. test support helpers like tests/utils/item.py) - real
|
|
31
|
+
# repos have both. Confirmed against tiangolo/full-stack-fastapi
|
|
32
|
+
# -template, which hits exactly this case.
|
|
33
|
+
"-x",
|
|
34
|
+
"*/test_*.py,*_test.py,*/tests/*,*/test/*",
|
|
35
|
+
],
|
|
36
|
+
capture_output=True,
|
|
37
|
+
text=True,
|
|
38
|
+
encoding="utf-8", # bandit emits UTF-8 JSON regardless of system locale
|
|
39
|
+
check=False, # bandit exits non-zero when it finds issues; expected
|
|
40
|
+
)
|
|
41
|
+
if not proc.stdout.strip():
|
|
42
|
+
if proc.returncode not in (0, 1):
|
|
43
|
+
raise RuntimeError(f"bandit failed to run: {proc.stderr.strip()}")
|
|
44
|
+
return []
|
|
45
|
+
|
|
46
|
+
raw = json.loads(proc.stdout)
|
|
47
|
+
findings: list[Finding] = []
|
|
48
|
+
for item in raw.get("results", []):
|
|
49
|
+
severity = _SEVERITY_MAP.get(item.get("issue_severity", "LOW"), "medium")
|
|
50
|
+
findings.append(
|
|
51
|
+
Finding(
|
|
52
|
+
tool="bandit",
|
|
53
|
+
language="python",
|
|
54
|
+
severity=severity,
|
|
55
|
+
file=item.get("filename", "unknown"),
|
|
56
|
+
line=item.get("line_number", 0),
|
|
57
|
+
message=item.get("issue_text", ""),
|
|
58
|
+
rule_id=item.get("test_id"),
|
|
59
|
+
)
|
|
60
|
+
)
|
|
61
|
+
return findings
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
"""Wraps `pip-audit -f json` into the shared Finding schema."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import json
|
|
5
|
+
import subprocess
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
from skeptic.core.models import Finding
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def run_pip_audit(repo_path: str) -> list[Finding]:
|
|
12
|
+
requirements_path = Path(repo_path) / "requirements.txt"
|
|
13
|
+
if requirements_path.exists():
|
|
14
|
+
cmd = ["pip-audit", "-f", "json", "-r", str(requirements_path)]
|
|
15
|
+
else:
|
|
16
|
+
# No requirements.txt - point pip-audit at the project directory
|
|
17
|
+
# itself so it can auto-detect a pyproject.toml/setup.py/setup.cfg
|
|
18
|
+
# and resolve dependencies directly. Most modern repos (uv, poetry,
|
|
19
|
+
# pdm) ship no requirements.txt at all; without this fallback they
|
|
20
|
+
# were silently treated as "0 vulnerabilities" when nothing had
|
|
21
|
+
# actually been checked. Confirmed against tiangolo/full-stack-
|
|
22
|
+
# fastapi-template, which is pyproject.toml/uv.lock-only.
|
|
23
|
+
cmd = ["pip-audit", "-f", "json", repo_path]
|
|
24
|
+
|
|
25
|
+
proc = subprocess.run(
|
|
26
|
+
cmd,
|
|
27
|
+
capture_output=True,
|
|
28
|
+
text=True,
|
|
29
|
+
encoding="utf-8", # pip-audit emits UTF-8 JSON regardless of system locale
|
|
30
|
+
check=False,
|
|
31
|
+
)
|
|
32
|
+
if not proc.stdout.strip():
|
|
33
|
+
# Either no supported project file was found (nothing to check) or
|
|
34
|
+
# no vulnerabilities were found - neither is a hard failure.
|
|
35
|
+
return []
|
|
36
|
+
|
|
37
|
+
try:
|
|
38
|
+
raw = json.loads(proc.stdout)
|
|
39
|
+
except json.JSONDecodeError:
|
|
40
|
+
return []
|
|
41
|
+
|
|
42
|
+
findings: list[Finding] = []
|
|
43
|
+
for dep in raw.get("dependencies", []):
|
|
44
|
+
for vuln in dep.get("vulns", []):
|
|
45
|
+
findings.append(
|
|
46
|
+
Finding(
|
|
47
|
+
tool="pip-audit",
|
|
48
|
+
language="python",
|
|
49
|
+
severity="high", # pip-audit doesn't provide severity by default; treat all as high
|
|
50
|
+
file="requirements.txt" if requirements_path.exists() else "pyproject.toml",
|
|
51
|
+
line=0,
|
|
52
|
+
message=f"{dep.get('name')} {dep.get('version')}: {vuln.get('id')} - {vuln.get('description', '')[:200]}",
|
|
53
|
+
rule_id=vuln.get("id"),
|
|
54
|
+
)
|
|
55
|
+
)
|
|
56
|
+
return findings
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
"""Wraps `pyright --outputjson` into the shared Finding schema."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import json
|
|
5
|
+
import subprocess
|
|
6
|
+
|
|
7
|
+
from skeptic.core.models import Finding
|
|
8
|
+
|
|
9
|
+
_SEVERITY_MAP = {
|
|
10
|
+
"error": "high",
|
|
11
|
+
"warning": "medium",
|
|
12
|
+
"information": "low",
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def run_pyright(repo_path: str) -> list[Finding]:
|
|
17
|
+
proc = subprocess.run(
|
|
18
|
+
["pyright", "--outputjson", repo_path],
|
|
19
|
+
capture_output=True,
|
|
20
|
+
text=True,
|
|
21
|
+
encoding="utf-8", # pyright emits UTF-8 JSON regardless of system locale -
|
|
22
|
+
# without this, Windows decodes with the locale codepage (e.g. cp1252) and
|
|
23
|
+
# any non-ASCII character in a diagnostic message (pyright uses U+00A0 for
|
|
24
|
+
# indentation) comes out as mojibake.
|
|
25
|
+
check=False, # pyright exits non-zero when it finds errors; expected
|
|
26
|
+
)
|
|
27
|
+
if not proc.stdout.strip():
|
|
28
|
+
raise RuntimeError(f"pyright produced no output: {proc.stderr.strip()}")
|
|
29
|
+
|
|
30
|
+
raw = json.loads(proc.stdout)
|
|
31
|
+
findings: list[Finding] = []
|
|
32
|
+
for item in raw.get("generalDiagnostics", []):
|
|
33
|
+
severity = _SEVERITY_MAP.get(item.get("severity", "warning"), "medium")
|
|
34
|
+
findings.append(
|
|
35
|
+
Finding(
|
|
36
|
+
tool="pyright",
|
|
37
|
+
language="python",
|
|
38
|
+
severity=severity,
|
|
39
|
+
file=item.get("file", "unknown"),
|
|
40
|
+
line=(item.get("range") or {}).get("start", {}).get("line", 0) + 1,
|
|
41
|
+
message=item.get("message", ""),
|
|
42
|
+
rule_id=item.get("rule"),
|
|
43
|
+
)
|
|
44
|
+
)
|
|
45
|
+
return findings
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
"""Runs pytest with coverage and returns a TestResult."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import json
|
|
5
|
+
import subprocess
|
|
6
|
+
import tempfile
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
from skeptic.core.models import TestResult
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def run_pytest(repo_path: str) -> TestResult:
|
|
13
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
14
|
+
cov_json_path = Path(tmp) / "coverage.json"
|
|
15
|
+
report_json_path = Path(tmp) / "report.json"
|
|
16
|
+
proc = subprocess.run(
|
|
17
|
+
[
|
|
18
|
+
"pytest",
|
|
19
|
+
repo_path,
|
|
20
|
+
"--cov=" + repo_path,
|
|
21
|
+
"--cov-report=json:" + str(cov_json_path),
|
|
22
|
+
"-q",
|
|
23
|
+
"--json-report",
|
|
24
|
+
"--json-report-file=" + str(report_json_path),
|
|
25
|
+
],
|
|
26
|
+
capture_output=True,
|
|
27
|
+
text=True,
|
|
28
|
+
encoding="utf-8", # pytest's own text output isn't parsed, but keep it
|
|
29
|
+
# consistent with the JSON files below, which are always UTF-8
|
|
30
|
+
check=False,
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
passed = 0
|
|
34
|
+
failed_count = 0
|
|
35
|
+
if report_json_path.exists():
|
|
36
|
+
try:
|
|
37
|
+
report_data = json.loads(report_json_path.read_text(encoding="utf-8"))
|
|
38
|
+
summary = report_data.get("summary", {})
|
|
39
|
+
passed = summary.get("passed", 0)
|
|
40
|
+
# "failed" only counts outright failures; treat errors the same way
|
|
41
|
+
# since both mean the suite did not fully succeed.
|
|
42
|
+
failed_count = summary.get("failed", 0) + summary.get("error", 0)
|
|
43
|
+
except (json.JSONDecodeError, OSError):
|
|
44
|
+
pass
|
|
45
|
+
elif proc.returncode not in (0, 1):
|
|
46
|
+
# pytest itself failed to run (bad args, collection crash, missing
|
|
47
|
+
# plugin, etc.) rather than just reporting failing tests - surface
|
|
48
|
+
# that instead of silently returning a fake all-zero result.
|
|
49
|
+
raise RuntimeError(f"pytest failed to run: {proc.stderr.strip() or proc.stdout.strip()}")
|
|
50
|
+
|
|
51
|
+
coverage_percent = 0.0
|
|
52
|
+
if cov_json_path.exists():
|
|
53
|
+
try:
|
|
54
|
+
cov_data = json.loads(cov_json_path.read_text(encoding="utf-8"))
|
|
55
|
+
coverage_percent = cov_data.get("totals", {}).get("percent_covered", 0.0)
|
|
56
|
+
except (json.JSONDecodeError, OSError):
|
|
57
|
+
pass
|
|
58
|
+
|
|
59
|
+
return TestResult(passed=passed, failed=failed_count, coverage_percent=coverage_percent)
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
"""Wraps `ruff check --output-format=json` into the shared Finding schema."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import json
|
|
5
|
+
import subprocess
|
|
6
|
+
|
|
7
|
+
from skeptic.core.models import Finding
|
|
8
|
+
|
|
9
|
+
# Ruff doesn't have severity levels itself; treat all lint findings as "medium"
|
|
10
|
+
# unless a rule code is one we know to be more serious.
|
|
11
|
+
_HIGH_SEVERITY_PREFIXES = ("S",) # flake8-bandit-equivalent rules, if enabled via ruff
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def run_ruff(repo_path: str) -> list[Finding]:
|
|
15
|
+
proc = subprocess.run(
|
|
16
|
+
["ruff", "check", repo_path, "--output-format=json"],
|
|
17
|
+
capture_output=True,
|
|
18
|
+
text=True,
|
|
19
|
+
encoding="utf-8", # ruff emits UTF-8 JSON regardless of system locale
|
|
20
|
+
check=False, # ruff exits non-zero when it finds issues; that's expected
|
|
21
|
+
)
|
|
22
|
+
if proc.returncode not in (0, 1):
|
|
23
|
+
raise RuntimeError(f"ruff failed to run: {proc.stderr.strip()}")
|
|
24
|
+
|
|
25
|
+
if not proc.stdout.strip():
|
|
26
|
+
return []
|
|
27
|
+
|
|
28
|
+
raw = json.loads(proc.stdout)
|
|
29
|
+
findings: list[Finding] = []
|
|
30
|
+
for item in raw:
|
|
31
|
+
code = item.get("code") or ""
|
|
32
|
+
severity = "high" if code.startswith(_HIGH_SEVERITY_PREFIXES) else "medium"
|
|
33
|
+
findings.append(
|
|
34
|
+
Finding(
|
|
35
|
+
tool="ruff",
|
|
36
|
+
language="python",
|
|
37
|
+
severity=severity,
|
|
38
|
+
file=item.get("filename", "unknown"),
|
|
39
|
+
line=(item.get("location") or {}).get("row", 0),
|
|
40
|
+
message=item.get("message", ""),
|
|
41
|
+
rule_id=code or None,
|
|
42
|
+
)
|
|
43
|
+
)
|
|
44
|
+
return findings
|
skeptic/cli/__init__.py
ADDED
|
File without changes
|
skeptic/cli/main.py
ADDED
|
@@ -0,0 +1,337 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import io
|
|
4
|
+
import json as json_module
|
|
5
|
+
import sys
|
|
6
|
+
|
|
7
|
+
import click
|
|
8
|
+
from rich.console import Console
|
|
9
|
+
from rich.table import Table
|
|
10
|
+
|
|
11
|
+
from skeptic.adapters.python import FINDING_ADAPTERS, TEST_ADAPTER
|
|
12
|
+
from skeptic.core.config import load_gate_config, write_starter_config
|
|
13
|
+
from skeptic.core.gate import evaluate_gate
|
|
14
|
+
from skeptic.core.risk_score import compute_risk_score
|
|
15
|
+
from skeptic.core.runner import run_all_checks
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _resilient_stdout():
|
|
19
|
+
"""Wraps stdout so an unencodable character degrades to "?" instead of
|
|
20
|
+
crashing the whole command with UnicodeEncodeError - the same failure
|
|
21
|
+
class fixed earlier for the PASS/FAIL marks (legacy Windows codepages
|
|
22
|
+
like cp1252 can't represent many Unicode characters), but general this
|
|
23
|
+
time: tool finding messages can contain non-ASCII characters (pyright
|
|
24
|
+
uses U+00A0 for indentation) and --narrate prints arbitrary LLM output
|
|
25
|
+
(which routinely includes em dashes, smart quotes, etc.). Falls back to
|
|
26
|
+
plain sys.stdout when there's no underlying buffer to wrap (e.g. under
|
|
27
|
+
pytest's capture, which already injects its own safe stream)."""
|
|
28
|
+
if not hasattr(sys.stdout, "buffer"):
|
|
29
|
+
return sys.stdout
|
|
30
|
+
return io.TextIOWrapper(sys.stdout.buffer, encoding=sys.stdout.encoding or "utf-8", errors="replace")
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
console = Console(file=_resilient_stdout())
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@click.group()
|
|
37
|
+
def cli():
|
|
38
|
+
"""skeptic - independent engineering quality gate for AI-generated code."""
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
@cli.command()
|
|
42
|
+
@click.argument("path", default=".")
|
|
43
|
+
@click.option("--json", "as_json", is_flag=True, help="Output machine-readable JSON instead of formatted text.")
|
|
44
|
+
@click.option(
|
|
45
|
+
"--narrate",
|
|
46
|
+
is_flag=True,
|
|
47
|
+
help="Explain SOLID findings in plain language via Gemini (opt-in: costs an API call per finding, "
|
|
48
|
+
"requires GEMINI_API_KEY, and never affects the pass/fail verdict - narration only explains, "
|
|
49
|
+
"it doesn't originate or override a finding).",
|
|
50
|
+
)
|
|
51
|
+
def check(path: str, as_json: bool, narrate: bool):
|
|
52
|
+
"""Run the quality gate against a repo (defaults to current directory)."""
|
|
53
|
+
report = run_all_checks(path, FINDING_ADAPTERS, TEST_ADAPTER)
|
|
54
|
+
gate_config = load_gate_config(path)
|
|
55
|
+
result = evaluate_gate(report, gate_config)
|
|
56
|
+
risk_score = compute_risk_score(report)
|
|
57
|
+
|
|
58
|
+
narrations = _narrate_architecture_findings(report) if narrate else {}
|
|
59
|
+
|
|
60
|
+
if as_json:
|
|
61
|
+
click.echo(
|
|
62
|
+
json_module.dumps(
|
|
63
|
+
{
|
|
64
|
+
"passed": result.passed,
|
|
65
|
+
"rules": [r.model_dump() for r in result.rules],
|
|
66
|
+
"tool_statuses": [s.model_dump() for s in report.tool_statuses],
|
|
67
|
+
# Not tied to any gate rule yet (Phase 3 milestone 8),
|
|
68
|
+
# so these don't show up under `rules` - surfaced here
|
|
69
|
+
# so they're visible at all in --json output.
|
|
70
|
+
"solid_findings": [f.model_dump() for f in report.findings if f.tool == "solid"],
|
|
71
|
+
"complexity_findings": [f.model_dump() for f in report.findings if f.tool == "complexity"],
|
|
72
|
+
"risk_score": risk_score.model_dump(),
|
|
73
|
+
"narrations": narrations,
|
|
74
|
+
},
|
|
75
|
+
default=str,
|
|
76
|
+
)
|
|
77
|
+
)
|
|
78
|
+
sys.exit(0 if result.passed else 1)
|
|
79
|
+
|
|
80
|
+
_print_report(result, report, risk_score, narrations)
|
|
81
|
+
sys.exit(0 if result.passed else 1)
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
@cli.command()
|
|
85
|
+
@click.argument("path", default=".")
|
|
86
|
+
@click.option(
|
|
87
|
+
"--target",
|
|
88
|
+
required=True,
|
|
89
|
+
help="Base URL of a running instance to test, e.g. http://localhost:8000. Must be a service you own "
|
|
90
|
+
"and control - the verifier sends real injection/auth-bypass/IDOR payloads at it.",
|
|
91
|
+
)
|
|
92
|
+
@click.option(
|
|
93
|
+
"--diff-ref",
|
|
94
|
+
default="HEAD",
|
|
95
|
+
show_default=True,
|
|
96
|
+
help="git diff ref to focus attack-case generation on (uncommitted changes vs this ref). "
|
|
97
|
+
"Falls back to general-purpose cases if `path` isn't a git repo or the ref doesn't exist.",
|
|
98
|
+
)
|
|
99
|
+
@click.option(
|
|
100
|
+
"--allow-external",
|
|
101
|
+
is_flag=True,
|
|
102
|
+
help="Allow targeting a non-local/non-private address. Only pass this against a service you own.",
|
|
103
|
+
)
|
|
104
|
+
@click.option("--max-cases", default=14, show_default=True, help="Maximum attack cases to generate.")
|
|
105
|
+
@click.option("--json", "as_json", is_flag=True, help="Output machine-readable JSON instead of formatted text.")
|
|
106
|
+
def verify(path: str, target: str, diff_ref: str, allow_external: bool, max_cases: int, as_json: bool):
|
|
107
|
+
"""Adversarial verifier: generates and runs attack test cases (boundary, invalid input,
|
|
108
|
+
injection, auth bypass, IDOR, concurrency, failure mode) against a running target you control.
|
|
109
|
+
|
|
110
|
+
Read-only against your code - never writes to `path`. Generates structured attack requests via
|
|
111
|
+
Gemini, never executable code; the only thing that ever runs is an HTTP request this tool sends
|
|
112
|
+
itself. Requires GEMINI_API_KEY (see README's narration section for setup - same key, same .env)."""
|
|
113
|
+
from skeptic.core.git_utils import get_diff
|
|
114
|
+
from skeptic.verifier import GenerationError, UnsafeTargetError, generate_attack_cases, is_local_or_private, run_attack_cases
|
|
115
|
+
|
|
116
|
+
# Check target safety before spending an API call on generation - no
|
|
117
|
+
# point paying for attack cases for a target we're about to refuse.
|
|
118
|
+
if not allow_external and not is_local_or_private(target):
|
|
119
|
+
console.print(
|
|
120
|
+
f"[red]'{target}' doesn't look like localhost or a private address. The verifier sends real "
|
|
121
|
+
"injection/auth-bypass/IDOR payloads - only run it against a service you own and control. "
|
|
122
|
+
"Pass --allow-external if you're certain this target is yours to test.[/red]"
|
|
123
|
+
)
|
|
124
|
+
sys.exit(1)
|
|
125
|
+
|
|
126
|
+
diff = get_diff(path, ref=diff_ref)
|
|
127
|
+
|
|
128
|
+
try:
|
|
129
|
+
cases = generate_attack_cases(diff, max_cases=max_cases)
|
|
130
|
+
except GenerationError as exc:
|
|
131
|
+
console.print(f"[red]Could not generate attack cases: {exc}[/red]")
|
|
132
|
+
sys.exit(1)
|
|
133
|
+
|
|
134
|
+
try:
|
|
135
|
+
report = run_attack_cases(target, cases, allow_external=allow_external)
|
|
136
|
+
except UnsafeTargetError as exc:
|
|
137
|
+
console.print(f"[red]{exc}[/red]")
|
|
138
|
+
sys.exit(1)
|
|
139
|
+
|
|
140
|
+
any_failed = any(r.passed is False for r in report.results)
|
|
141
|
+
|
|
142
|
+
if as_json:
|
|
143
|
+
click.echo(json_module.dumps(report.model_dump(), default=str))
|
|
144
|
+
sys.exit(1 if any_failed else 0)
|
|
145
|
+
|
|
146
|
+
_print_verification_report(report)
|
|
147
|
+
sys.exit(1 if any_failed else 0)
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
@cli.command()
|
|
151
|
+
@click.argument("path", default=".")
|
|
152
|
+
@click.option("--since-ref", default="HEAD~20", show_default=True, help="Start of the commit range to analyze.")
|
|
153
|
+
@click.option("--until-ref", default="HEAD", show_default=True, help="End of the commit range to analyze.")
|
|
154
|
+
@click.option(
|
|
155
|
+
"--window-minutes",
|
|
156
|
+
default=15,
|
|
157
|
+
show_default=True,
|
|
158
|
+
help="A commit counts as AI-influenced if a logged MCP call for this repo landed within this many "
|
|
159
|
+
"minutes of it.",
|
|
160
|
+
)
|
|
161
|
+
@click.option("--json", "as_json", is_flag=True, help="Output machine-readable JSON instead of formatted text.")
|
|
162
|
+
def provenance(path: str, since_ref: str, until_ref: str, window_minutes: int, as_json: bool):
|
|
163
|
+
"""AI provenance tagging: estimates what share of recent commits landed while an AI agent
|
|
164
|
+
was actively using skeptic-mcp against this repo (Phase 3 milestone 7).
|
|
165
|
+
|
|
166
|
+
An approximation, not a precise record: skeptic-mcp's tools are read-only analysis, so the
|
|
167
|
+
call log records when an agent called them, not which lines it edited. "ai_influenced" means
|
|
168
|
+
a commit landed within --window-minutes of a logged call - a timing correlation, not proof of
|
|
169
|
+
authorship. Requires no MCP history to run; with none, everything is reported as human."""
|
|
170
|
+
from skeptic.core.provenance import tag_provenance
|
|
171
|
+
|
|
172
|
+
report = tag_provenance(path, since_ref=since_ref, until_ref=until_ref, window_minutes=window_minutes)
|
|
173
|
+
|
|
174
|
+
if as_json:
|
|
175
|
+
click.echo(json_module.dumps(report.model_dump(), default=str))
|
|
176
|
+
sys.exit(0)
|
|
177
|
+
|
|
178
|
+
_print_provenance_report(report)
|
|
179
|
+
sys.exit(0)
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
@cli.command()
|
|
183
|
+
@click.argument("path", default=".")
|
|
184
|
+
def init(path: str):
|
|
185
|
+
"""Write a starter skeptic.yaml into the given directory."""
|
|
186
|
+
try:
|
|
187
|
+
config_path = write_starter_config(path)
|
|
188
|
+
console.print(f"[green]Created {config_path}[/green]")
|
|
189
|
+
except FileExistsError as exc:
|
|
190
|
+
console.print(f"[yellow]{exc}[/yellow]")
|
|
191
|
+
sys.exit(1)
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def _finding_key(finding) -> str:
|
|
195
|
+
return f"{finding.file}:{finding.line}:{finding.rule_id}"
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def _architecture_findings(report) -> list:
|
|
199
|
+
"""solid + complexity findings together - both are "how sound is this
|
|
200
|
+
code's structure", surfaced as one section/gate the same way
|
|
201
|
+
core.gate's "architecture" rule counts them together."""
|
|
202
|
+
return [f for f in report.findings if f.tool in ("solid", "complexity")]
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def _narrate_architecture_findings(report) -> dict:
|
|
206
|
+
"""Best-effort: narration is supplementary, so a failure here (missing
|
|
207
|
+
key, network error, ...) must never crash `skeptic check` - it degrades
|
|
208
|
+
to a per-finding error message instead."""
|
|
209
|
+
findings = _architecture_findings(report)
|
|
210
|
+
if not findings:
|
|
211
|
+
return {}
|
|
212
|
+
|
|
213
|
+
# Deferred import: google-genai is an optional extra (pyproject.toml's
|
|
214
|
+
# `narration` group) - importing it here means users who never pass
|
|
215
|
+
# --narrate never need it installed.
|
|
216
|
+
from skeptic.narration import NarrationError, narrate_finding
|
|
217
|
+
|
|
218
|
+
narrations: dict[str, str] = {}
|
|
219
|
+
for finding in findings:
|
|
220
|
+
try:
|
|
221
|
+
narrations[_finding_key(finding)] = narrate_finding(finding)
|
|
222
|
+
except NarrationError as exc:
|
|
223
|
+
narrations[_finding_key(finding)] = f"[narration unavailable: {exc}]"
|
|
224
|
+
return narrations
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def _print_report(result, report, risk_score=None, narrations: dict | None = None):
|
|
228
|
+
narrations = narrations or {}
|
|
229
|
+
status = "[bold green]PASS[/bold green]" if result.passed else "[bold red]FAIL[/bold red]"
|
|
230
|
+
console.print(f"\nEngineering Gate: {status}\n")
|
|
231
|
+
|
|
232
|
+
if risk_score is not None:
|
|
233
|
+
label_color = {"LOW": "green", "MEDIUM": "yellow", "HIGH": "red"}[risk_score.label]
|
|
234
|
+
dims = ", ".join(f"{name}={value}" for name, value in risk_score.dimensions.items())
|
|
235
|
+
console.print(
|
|
236
|
+
f"Change Risk Score: [{label_color}]{risk_score.label}[/{label_color}] "
|
|
237
|
+
f"({risk_score.composite_score}/100 - {dims})\n"
|
|
238
|
+
)
|
|
239
|
+
|
|
240
|
+
table = Table(show_header=True, header_style="bold")
|
|
241
|
+
table.add_column("Rule")
|
|
242
|
+
table.add_column("Status")
|
|
243
|
+
table.add_column("Detail")
|
|
244
|
+
|
|
245
|
+
for rule in result.rules:
|
|
246
|
+
# Plain ASCII, not unicode check/cross marks: on Windows, terminals
|
|
247
|
+
# stuck on a legacy codepage (cp1252 etc., still common outside
|
|
248
|
+
# Windows Terminal/PowerShell 7) can't encode \u2713/\u2717 and rich
|
|
249
|
+
# crashes the whole command trying to print them.
|
|
250
|
+
mark = "[green]PASS[/green]" if rule.passed else "[red]FAIL[/red]"
|
|
251
|
+
table.add_row(rule.name, mark, rule.detail)
|
|
252
|
+
|
|
253
|
+
console.print(table)
|
|
254
|
+
|
|
255
|
+
failed_tools = [s for s in report.tool_statuses if not s.ok]
|
|
256
|
+
if failed_tools:
|
|
257
|
+
console.print("\n[yellow]Warning: some tools failed to run and were skipped:[/yellow]")
|
|
258
|
+
for s in failed_tools:
|
|
259
|
+
console.print(f" - {s.tool}: {s.error}")
|
|
260
|
+
|
|
261
|
+
for rule in result.rules:
|
|
262
|
+
if not rule.passed and rule.violating_findings:
|
|
263
|
+
console.print(f"\n[bold]{rule.name} findings:[/bold]")
|
|
264
|
+
for f in rule.violating_findings[:10]:
|
|
265
|
+
console.print(f" {f.file}:{f.line} [{f.severity}] {f.message} ({f.rule_id})")
|
|
266
|
+
if len(rule.violating_findings) > 10:
|
|
267
|
+
console.print(f" ... and {len(rule.violating_findings) - 10} more")
|
|
268
|
+
|
|
269
|
+
# Shown here regardless of whether the "architecture" gate rule ran
|
|
270
|
+
# above (it only appears in the table if architecture_max_findings is
|
|
271
|
+
# configured) - these findings are useful context either way, and this
|
|
272
|
+
# is the only place they're visible if the rule isn't turned on.
|
|
273
|
+
arch_findings = _architecture_findings(report)
|
|
274
|
+
if arch_findings:
|
|
275
|
+
gated = any(r.name == "architecture" for r in result.rules)
|
|
276
|
+
label = "architecture findings" if gated else "architecture findings (informational - not gated; see skeptic.yaml)"
|
|
277
|
+
console.print(f"\n[bold]{label}:[/bold]")
|
|
278
|
+
for f in arch_findings:
|
|
279
|
+
console.print(f" {f.file}:{f.line} [{f.severity}] {f.message} ({f.rule_id})")
|
|
280
|
+
narration = narrations.get(_finding_key(f))
|
|
281
|
+
if narration:
|
|
282
|
+
console.print(f" [dim]narration:[/dim] {narration}")
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
def _print_verification_report(report):
|
|
286
|
+
console.print(f"\nAdversarial verification against {report.target}\n")
|
|
287
|
+
|
|
288
|
+
table = Table(show_header=True, header_style="bold")
|
|
289
|
+
table.add_column("Category")
|
|
290
|
+
table.add_column("Passed")
|
|
291
|
+
table.add_column("Failed")
|
|
292
|
+
table.add_column("Review")
|
|
293
|
+
|
|
294
|
+
for category, counts in sorted(report.summary_by_category().items()):
|
|
295
|
+
table.add_row(category, str(counts["passed"]), str(counts["failed"]), str(counts["review"]))
|
|
296
|
+
|
|
297
|
+
console.print(table)
|
|
298
|
+
|
|
299
|
+
failing = [r for r in report.results if r.passed is False]
|
|
300
|
+
if failing:
|
|
301
|
+
console.print("\n[bold red]Findings that look vulnerable or crashed:[/bold red]")
|
|
302
|
+
for r in failing:
|
|
303
|
+
console.print(f" [{r.case.category}] {r.case.method} {r.case.path} -> {r.status_code}: {r.detail}")
|
|
304
|
+
|
|
305
|
+
review = [r for r in report.results if r.passed is None]
|
|
306
|
+
if review:
|
|
307
|
+
console.print(f"\n[yellow]{len(review)} result(s) need manual review (heuristic couldn't tell):[/yellow]")
|
|
308
|
+
for r in review[:10]:
|
|
309
|
+
console.print(f" [{r.case.category}] {r.case.method} {r.case.path} -> {r.status_code}: {r.detail}")
|
|
310
|
+
if len(review) > 10:
|
|
311
|
+
console.print(f" ... and {len(review) - 10} more")
|
|
312
|
+
|
|
313
|
+
|
|
314
|
+
def _print_provenance_report(report):
|
|
315
|
+
console.print(f"\nAI provenance for {report.repo_path}\n")
|
|
316
|
+
console.print(f"Commits analyzed: {report.commits_analyzed} (window: {report.window_minutes} min)")
|
|
317
|
+
console.print(
|
|
318
|
+
f"AI-influenced line ratio: {report.ai_influenced_line_ratio:.1%} "
|
|
319
|
+
"[dim](timing correlation with MCP call history, not exact per-line attribution)[/dim]\n"
|
|
320
|
+
)
|
|
321
|
+
|
|
322
|
+
counts = report.label_counts()
|
|
323
|
+
table = Table(show_header=True, header_style="bold")
|
|
324
|
+
table.add_column("Label")
|
|
325
|
+
table.add_column("Commits")
|
|
326
|
+
for label in ("human", "ai_generated", "ai_modified"):
|
|
327
|
+
table.add_row(label, str(counts[label]))
|
|
328
|
+
console.print(table)
|
|
329
|
+
|
|
330
|
+
if report.commits_analyzed == 0:
|
|
331
|
+
console.print(
|
|
332
|
+
"\n[dim]No commits in range - try a wider --since-ref, or confirm this is a git repo.[/dim]"
|
|
333
|
+
)
|
|
334
|
+
|
|
335
|
+
|
|
336
|
+
if __name__ == "__main__":
|
|
337
|
+
cli()
|
skeptic/core/__init__.py
ADDED
|
File without changes
|