verdity 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- verdity/__init__.py +6 -0
- verdity/agents/__init__.py +17 -0
- verdity/agents/base.py +105 -0
- verdity/agents/code_quality.py +92 -0
- verdity/agents/documentation.py +74 -0
- verdity/agents/security.py +274 -0
- verdity/agents/testing.py +74 -0
- verdity/aggregator.py +158 -0
- verdity/approval_queue.py +156 -0
- verdity/async_sqlite.py +69 -0
- verdity/audit_store.py +133 -0
- verdity/budget_enforcer.py +184 -0
- verdity/coding_agent.py +166 -0
- verdity/config.py +87 -0
- verdity/event_queue.py +214 -0
- verdity/gateway/__init__.py +7 -0
- verdity/gateway/app.py +253 -0
- verdity/github_client.py +256 -0
- verdity/hmac_verify.py +60 -0
- verdity/orchestrator.py +377 -0
- verdity/rate_limiter.py +192 -0
- verdity/router.py +116 -0
- verdity/schemas/__init__.py +39 -0
- verdity/schemas/_models.py +176 -0
- verdity/semantic_index.py +429 -0
- verdity/token_economics.py +264 -0
- verdity/verification_gate.py +254 -0
- verdity/webhook_normalizer.py +98 -0
- verdity/worker.py +220 -0
- verdity-0.2.0.dist-info/METADATA +359 -0
- verdity-0.2.0.dist-info/RECORD +35 -0
- verdity-0.2.0.dist-info/WHEEL +5 -0
- verdity-0.2.0.dist-info/entry_points.txt +2 -0
- verdity-0.2.0.dist-info/licenses/LICENSE +21 -0
- verdity-0.2.0.dist-info/top_level.txt +1 -0
verdity/__init__.py
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Verdity specialist agents package.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from verdity.agents.base import BaseSpecialistAgent
|
|
6
|
+
from verdity.agents.code_quality import CodeQualityAgent
|
|
7
|
+
from verdity.agents.documentation import DocumentationAgent
|
|
8
|
+
from verdity.agents.security import SecurityAgent
|
|
9
|
+
from verdity.agents.testing import TestingAgent
|
|
10
|
+
|
|
11
|
+
__all__ = [
|
|
12
|
+
"BaseSpecialistAgent",
|
|
13
|
+
"CodeQualityAgent",
|
|
14
|
+
"DocumentationAgent",
|
|
15
|
+
"SecurityAgent",
|
|
16
|
+
"TestingAgent",
|
|
17
|
+
]
|
verdity/agents/base.py
ADDED
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Base specialist agent — shared boilerplate for all Verdity specialist agents.
|
|
3
|
+
|
|
4
|
+
Eliminates duplication across security, code_quality, testing, and documentation
|
|
5
|
+
agents by providing common run orchestration, token metering, and audit logging.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import hashlib
|
|
11
|
+
import logging
|
|
12
|
+
from abc import ABC, abstractmethod
|
|
13
|
+
|
|
14
|
+
from verdity.audit_store import AuditStore
|
|
15
|
+
from verdity.schemas import (
|
|
16
|
+
ConcernType,
|
|
17
|
+
Finding,
|
|
18
|
+
SpecialistContext,
|
|
19
|
+
SpecialistResponse,
|
|
20
|
+
)
|
|
21
|
+
from verdity.semantic_index import SemanticIndex
|
|
22
|
+
from verdity.token_economics import TokenEconomicsService
|
|
23
|
+
|
|
24
|
+
logger = logging.getLogger(__name__)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class BaseSpecialistAgent(ABC):
|
|
28
|
+
"""Abstract base for all specialist agents. Handles metering and audit."""
|
|
29
|
+
|
|
30
|
+
AGENT_VERSION: str = "base-agent@0.0.0"
|
|
31
|
+
SPECIALIST_NAME: str = "base"
|
|
32
|
+
CONCERN_TYPE: ConcernType = ConcernType.SECURITY
|
|
33
|
+
|
|
34
|
+
# Subclasses override these for token estimation
|
|
35
|
+
_input_tokens_per_finding: int = 300
|
|
36
|
+
_output_tokens_per_finding: int = 50
|
|
37
|
+
|
|
38
|
+
async def run(
|
|
39
|
+
self,
|
|
40
|
+
ctx: SpecialistContext,
|
|
41
|
+
semantic_index: SemanticIndex,
|
|
42
|
+
token_economics: TokenEconomicsService,
|
|
43
|
+
audit_store: AuditStore,
|
|
44
|
+
) -> SpecialistResponse:
|
|
45
|
+
"""
|
|
46
|
+
Template method: scan → record tokens → audit-log findings → return.
|
|
47
|
+
Subclasses implement `_scan()` for the actual analysis.
|
|
48
|
+
"""
|
|
49
|
+
findings = await self._scan(ctx, semantic_index)
|
|
50
|
+
|
|
51
|
+
input_tokens = len(findings) * self._input_tokens_per_finding
|
|
52
|
+
output_tokens = len(findings) * self._output_tokens_per_finding
|
|
53
|
+
|
|
54
|
+
await token_economics.record_call(
|
|
55
|
+
review_run_id=ctx.review_run_id,
|
|
56
|
+
agent_name=self.AGENT_VERSION,
|
|
57
|
+
model=f"{self.SPECIALIST_NAME}/dev",
|
|
58
|
+
input_tokens=input_tokens,
|
|
59
|
+
output_tokens=output_tokens,
|
|
60
|
+
repo_owner=ctx.repo_owner,
|
|
61
|
+
repo_name=ctx.repo_name,
|
|
62
|
+
org=ctx.repo_owner,
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
for finding in findings:
|
|
66
|
+
await audit_store.append(
|
|
67
|
+
event_type="finding.created",
|
|
68
|
+
entity_type="finding",
|
|
69
|
+
entity_id=str(finding.finding_id),
|
|
70
|
+
payload={
|
|
71
|
+
"concern": finding.concern.value,
|
|
72
|
+
"severity": finding.severity.value,
|
|
73
|
+
"file": finding.file,
|
|
74
|
+
"summary": finding.summary,
|
|
75
|
+
"confidence": finding.confidence,
|
|
76
|
+
"agent_version": self.AGENT_VERSION,
|
|
77
|
+
},
|
|
78
|
+
related_run_id=ctx.review_run_id,
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
logger.info(
|
|
82
|
+
"%s run %s: %d findings",
|
|
83
|
+
self.SPECIALIST_NAME, ctx.review_run_id, len(findings),
|
|
84
|
+
)
|
|
85
|
+
return SpecialistResponse(
|
|
86
|
+
review_run_id=ctx.review_run_id,
|
|
87
|
+
specialist=self.SPECIALIST_NAME,
|
|
88
|
+
status="complete",
|
|
89
|
+
findings=findings,
|
|
90
|
+
tokens_used={"input": input_tokens, "output": output_tokens},
|
|
91
|
+
cost_usd=0.0,
|
|
92
|
+
)
|
|
93
|
+
|
|
94
|
+
@abstractmethod
|
|
95
|
+
async def _scan(
|
|
96
|
+
self,
|
|
97
|
+
ctx: SpecialistContext,
|
|
98
|
+
semantic_index: SemanticIndex,
|
|
99
|
+
) -> list[Finding]:
|
|
100
|
+
"""Subclass implements the actual scan logic."""
|
|
101
|
+
...
|
|
102
|
+
|
|
103
|
+
@staticmethod
|
|
104
|
+
def _prompt_hash(*parts: str) -> str:
|
|
105
|
+
return "sha256:" + hashlib.sha256("|".join(parts).encode()).hexdigest()[:16]
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Code Quality Specialist Agent.
|
|
3
|
+
|
|
4
|
+
Reviews PR diffs for code style, maintainability, and structural issues.
|
|
5
|
+
Produces schema-valid findings with deterministic confidence scores.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import logging
|
|
11
|
+
|
|
12
|
+
from verdity.agents.base import BaseSpecialistAgent
|
|
13
|
+
from verdity.schemas import (
|
|
14
|
+
ConcernType,
|
|
15
|
+
EvidenceItem,
|
|
16
|
+
Finding,
|
|
17
|
+
SpecialistContext,
|
|
18
|
+
Severity,
|
|
19
|
+
)
|
|
20
|
+
from verdity.semantic_index import SemanticIndex
|
|
21
|
+
|
|
22
|
+
logger = logging.getLogger(__name__)
|
|
23
|
+
|
|
24
|
+
# ── Code quality patterns (deterministic rules) ──────────────────────
|
|
25
|
+
|
|
26
|
+
QUALITY_PATTERNS: list[tuple[str, str, str, str]] = [
|
|
27
|
+
# (name, pattern, severity, explanation)
|
|
28
|
+
("long_function", "def .*\\(.*\\):\\n.{200,}", "medium", "Function may be too long — consider refactoring"),
|
|
29
|
+
("deep_nesting", " {5,}", "low", "Deep nesting reduces readability — consider early returns"),
|
|
30
|
+
("todo_comment", "TODO", "info", "TODO comment found — track for follow-up"),
|
|
31
|
+
("fixme_comment", "FIXME", "low", "FIXME comment found — should be addressed before merge"),
|
|
32
|
+
("magic_number", "= \\d{4,}", "low", "Magic number — consider a named constant"),
|
|
33
|
+
("bare_except", "except:", "medium", "Bare except catches all exceptions — specify exception types"),
|
|
34
|
+
("global_import", "from .* import \\*", "high", "Wildcard import pollutes namespace — import names explicitly"),
|
|
35
|
+
("assert_in_code", "assert ", "medium", "Assert statement in production code — removes on optimize"),
|
|
36
|
+
("print_statement", "print\\(", "low", "Debug print statement — remove before merge"),
|
|
37
|
+
]
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class CodeQualityAgent(BaseSpecialistAgent):
|
|
41
|
+
"""Code quality specialist agent for Verdity."""
|
|
42
|
+
|
|
43
|
+
AGENT_VERSION = "code-quality-agent@0.1.0"
|
|
44
|
+
SPECIALIST_NAME = "code_quality"
|
|
45
|
+
CONCERN_TYPE = ConcernType.CODE_QUALITY
|
|
46
|
+
_input_tokens_per_finding = 300
|
|
47
|
+
_output_tokens_per_finding = 50
|
|
48
|
+
|
|
49
|
+
async def _scan(
|
|
50
|
+
self,
|
|
51
|
+
ctx: SpecialistContext,
|
|
52
|
+
semantic_index: SemanticIndex,
|
|
53
|
+
) -> list[Finding]:
|
|
54
|
+
findings: list[Finding] = []
|
|
55
|
+
|
|
56
|
+
for file_info in ctx.diff_files:
|
|
57
|
+
path = file_info.get("path", "")
|
|
58
|
+
content = file_info.get("content", "")
|
|
59
|
+
additions = file_info.get("additions", "")
|
|
60
|
+
scan_text = additions if additions else content
|
|
61
|
+
|
|
62
|
+
for name, pattern, severity, explanation in QUALITY_PATTERNS:
|
|
63
|
+
if pattern.lower() in scan_text.lower():
|
|
64
|
+
lines = scan_text.split("\n")
|
|
65
|
+
for i, line in enumerate(lines, start=1):
|
|
66
|
+
if pattern.lower() in line.lower():
|
|
67
|
+
findings.append(Finding(
|
|
68
|
+
concern=ConcernType.CODE_QUALITY,
|
|
69
|
+
severity=Severity(severity),
|
|
70
|
+
file=path,
|
|
71
|
+
line_start=i,
|
|
72
|
+
line_end=i,
|
|
73
|
+
summary=f"{name.replace('_', ' ').title()} detected",
|
|
74
|
+
explanation=f"{explanation} at {path}:{i}",
|
|
75
|
+
suggested_fix_diff=self._suggested_fix(name),
|
|
76
|
+
confidence=0.6 if severity == "info" else 0.7,
|
|
77
|
+
evidence=[EvidenceItem(tool="code_quality_linter", query=pattern, result=name)],
|
|
78
|
+
agent_version=self.AGENT_VERSION,
|
|
79
|
+
prompt_hash=self._prompt_hash(name, path, str(i)),
|
|
80
|
+
))
|
|
81
|
+
break
|
|
82
|
+
|
|
83
|
+
return findings
|
|
84
|
+
|
|
85
|
+
@staticmethod
|
|
86
|
+
def _suggested_fix(name: str) -> str | None:
|
|
87
|
+
fixes = {
|
|
88
|
+
"bare_except": "- except:\n+ except Exception as e:\n logger.error(...)",
|
|
89
|
+
"global_import": "- from module import *\n+ from module import specific_name",
|
|
90
|
+
"print_statement": "- print(x)\n+ logging.debug(%r, x)",
|
|
91
|
+
}
|
|
92
|
+
return fixes.get(name)
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Documentation Specialist Agent.
|
|
3
|
+
|
|
4
|
+
Reviews PR diffs for documentation quality and completeness.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import logging
|
|
10
|
+
|
|
11
|
+
from verdity.agents.base import BaseSpecialistAgent
|
|
12
|
+
from verdity.schemas import (
|
|
13
|
+
ConcernType,
|
|
14
|
+
EvidenceItem,
|
|
15
|
+
Finding,
|
|
16
|
+
SpecialistContext,
|
|
17
|
+
Severity,
|
|
18
|
+
)
|
|
19
|
+
from verdity.semantic_index import SemanticIndex
|
|
20
|
+
|
|
21
|
+
logger = logging.getLogger(__name__)
|
|
22
|
+
|
|
23
|
+
DOC_PATTERNS: list[tuple[str, str, str, str]] = [
|
|
24
|
+
("missing_docstring", "def ", "info", "Function added — verify docstring coverage"),
|
|
25
|
+
("missing_type_hints", "def ", "low", "Function may lack type hints"),
|
|
26
|
+
("changelog_entry", "CHANGELOG", "info", "Changelog entry detected — verify format"),
|
|
27
|
+
("breaking_change", "breaking change", "medium", "Potential breaking change noted — verify migration docs"),
|
|
28
|
+
]
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class DocumentationAgent(BaseSpecialistAgent):
|
|
32
|
+
"""Documentation specialist agent for Verdity."""
|
|
33
|
+
|
|
34
|
+
AGENT_VERSION = "docs-agent@0.1.0"
|
|
35
|
+
SPECIALIST_NAME = "documentation"
|
|
36
|
+
CONCERN_TYPE = ConcernType.DOCUMENTATION
|
|
37
|
+
_input_tokens_per_finding = 150
|
|
38
|
+
_output_tokens_per_finding = 20
|
|
39
|
+
|
|
40
|
+
async def _scan(
|
|
41
|
+
self,
|
|
42
|
+
ctx: SpecialistContext,
|
|
43
|
+
semantic_index: SemanticIndex,
|
|
44
|
+
) -> list[Finding]:
|
|
45
|
+
findings: list[Finding] = []
|
|
46
|
+
|
|
47
|
+
for file_info in ctx.diff_files:
|
|
48
|
+
path = file_info.get("path", "")
|
|
49
|
+
content = file_info.get("content", "")
|
|
50
|
+
additions = file_info.get("additions", "")
|
|
51
|
+
scan_text = additions if additions else content
|
|
52
|
+
|
|
53
|
+
for name, pattern, severity, explanation in DOC_PATTERNS:
|
|
54
|
+
if pattern.lower() in scan_text.lower():
|
|
55
|
+
lines = scan_text.split("\n")
|
|
56
|
+
for i, line in enumerate(lines, start=1):
|
|
57
|
+
if pattern.lower() in line.lower():
|
|
58
|
+
findings.append(Finding(
|
|
59
|
+
concern=ConcernType.DOCUMENTATION,
|
|
60
|
+
severity=Severity(severity),
|
|
61
|
+
file=path,
|
|
62
|
+
line_start=i,
|
|
63
|
+
line_end=i,
|
|
64
|
+
summary=f"{name.replace('_', ' ').title()}",
|
|
65
|
+
explanation=f"{explanation} at {path}:{i}",
|
|
66
|
+
suggested_fix_diff=None,
|
|
67
|
+
confidence=0.4 if severity == "info" else 0.55,
|
|
68
|
+
evidence=[EvidenceItem(tool="docs_analyzer", query=pattern, result=name)],
|
|
69
|
+
agent_version=self.AGENT_VERSION,
|
|
70
|
+
prompt_hash=self._prompt_hash(name, path, str(i)),
|
|
71
|
+
))
|
|
72
|
+
break
|
|
73
|
+
|
|
74
|
+
return findings
|
|
@@ -0,0 +1,274 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Security Specialist Agent.
|
|
3
|
+
|
|
4
|
+
Produces structured, evidence-backed security findings for PR diffs.
|
|
5
|
+
Tools available: semantic_search, secret_scanner, cve_lookup (mocked in dev).
|
|
6
|
+
|
|
7
|
+
Non-negotiable constraints satisfied:
|
|
8
|
+
- #4: Uses the shared SemanticIndex (no private vector store)
|
|
9
|
+
- #5: Confidence computed by deterministic post-processing, never raw LLM self-report
|
|
10
|
+
- #8: Every model/tool call metered through TokenEconomicsService
|
|
11
|
+
- #9: Every finding logged to Audit Store
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import logging
|
|
17
|
+
|
|
18
|
+
from verdity.agents.base import BaseSpecialistAgent
|
|
19
|
+
from verdity.schemas import (
|
|
20
|
+
ConcernType,
|
|
21
|
+
EvidenceItem,
|
|
22
|
+
Finding,
|
|
23
|
+
Severity,
|
|
24
|
+
SpecialistContext,
|
|
25
|
+
)
|
|
26
|
+
from verdity.semantic_index import SemanticIndex
|
|
27
|
+
|
|
28
|
+
logger = logging.getLogger(__name__)
|
|
29
|
+
|
|
30
|
+
# ── Security scan patterns (deterministic rules, no LLM needed for these) ──
|
|
31
|
+
|
|
32
|
+
_SECRET_PATTERNS: list[tuple[str, str, str]] = [
|
|
33
|
+
# (name, regex-like substring, severity)
|
|
34
|
+
("AWS_ACCESS_KEY", "aws_access_key", "high"),
|
|
35
|
+
("AWS_SECRET_KEY", "aws_secret_key", "high"),
|
|
36
|
+
("PRIVATE_KEY", "-----BEGIN.*PRIVATE KEY-----", "critical"),
|
|
37
|
+
("GITHUB_TOKEN", "ghp_", "high"),
|
|
38
|
+
("GITHUB_TOKEN_ALT", "github_pat_", "high"),
|
|
39
|
+
("SLACK_TOKEN", "xoxb-", "medium"),
|
|
40
|
+
("SLACK_TOKEN_USER", "xoxp-", "medium"),
|
|
41
|
+
("JWT_SECRET", "jwtSecret", "high"),
|
|
42
|
+
("API_KEY_ASSIGN", '= "sk-', "high"),
|
|
43
|
+
("HARDCODED_PASSWORD", "password = '", "high"),
|
|
44
|
+
]
|
|
45
|
+
|
|
46
|
+
_CWE_MAPPING: dict[str, str] = {
|
|
47
|
+
"AWS_ACCESS_KEY": "CWE-798 (Use of Hard-coded Credentials)",
|
|
48
|
+
"PRIVATE_KEY": "CWE-321 (Use of Hard-coded Cryptographic Key)",
|
|
49
|
+
"GITHUB_TOKEN": "CWE-798",
|
|
50
|
+
"GITHUB_TOKEN_ALT": "CWE-798",
|
|
51
|
+
"SLACK_TOKEN": "CWE-798",
|
|
52
|
+
"JWT_SECRET": "CWE-798",
|
|
53
|
+
"API_KEY_ASSIGN": "CWE-798",
|
|
54
|
+
"HARDCODED_PASSWORD": "CWE-798",
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
class SecurityAgent(BaseSpecialistAgent):
|
|
59
|
+
"""
|
|
60
|
+
Security specialist agent for Verdity.
|
|
61
|
+
|
|
62
|
+
Performs deterministic static analysis + semantic search to produce
|
|
63
|
+
security findings with cited evidence and computed confidence scores.
|
|
64
|
+
"""
|
|
65
|
+
|
|
66
|
+
AGENT_VERSION = "security-agent@0.1.0"
|
|
67
|
+
SPECIALIST_NAME = "security"
|
|
68
|
+
CONCERN_TYPE = ConcernType.SECURITY
|
|
69
|
+
_input_tokens_per_finding = 500
|
|
70
|
+
_output_tokens_per_finding = 50
|
|
71
|
+
|
|
72
|
+
async def _scan(
|
|
73
|
+
self,
|
|
74
|
+
ctx: SpecialistContext,
|
|
75
|
+
semantic_index: SemanticIndex,
|
|
76
|
+
) -> list[Finding]:
|
|
77
|
+
findings: list[Finding] = []
|
|
78
|
+
|
|
79
|
+
# ── Pass 1: Deterministic rule-based scans (no LLM cost) ──────
|
|
80
|
+
rule_findings = self._scan_for_secrets(ctx.diff_files)
|
|
81
|
+
findings.extend(rule_findings)
|
|
82
|
+
|
|
83
|
+
# ── Pass 2: Semantic search for security-relevant patterns ────
|
|
84
|
+
security_queries = [
|
|
85
|
+
"authentication", "authorization", "session", "token",
|
|
86
|
+
"password", "crypto", "hash", "encryption", "SQL injection",
|
|
87
|
+
"XSS", "CSRF", "deserialization", "file upload",
|
|
88
|
+
]
|
|
89
|
+
semantic_findings = await self._semantic_security_search(
|
|
90
|
+
ctx=ctx,
|
|
91
|
+
queries=security_queries,
|
|
92
|
+
semantic_index=semantic_index,
|
|
93
|
+
)
|
|
94
|
+
findings.extend(semantic_findings)
|
|
95
|
+
|
|
96
|
+
# ── Pass 3: Diff-aware analysis (check added lines specifically) ─
|
|
97
|
+
diff_findings = self._scan_diff_for_vulnerabilities(ctx.diff_files)
|
|
98
|
+
findings.extend(diff_findings)
|
|
99
|
+
|
|
100
|
+
return findings
|
|
101
|
+
|
|
102
|
+
# ── Rule-Based Scans ──────────────────────────────────────────────
|
|
103
|
+
|
|
104
|
+
def _scan_for_secrets(self, diff_files: list[dict]) -> list[Finding]:
|
|
105
|
+
"""Deterministic scan for hard-coded secrets in diff content."""
|
|
106
|
+
findings: list[Finding] = []
|
|
107
|
+
for file_info in diff_files:
|
|
108
|
+
path = file_info.get("path", "")
|
|
109
|
+
content = file_info.get("content", "")
|
|
110
|
+
additions = file_info.get("additions", "")
|
|
111
|
+
|
|
112
|
+
scan_text = additions if additions else content
|
|
113
|
+
|
|
114
|
+
for pattern_name, pattern_str, severity_str in _SECRET_PATTERNS:
|
|
115
|
+
if pattern_str.lower() in scan_text.lower():
|
|
116
|
+
lines = scan_text.split("\n")
|
|
117
|
+
for i, line in enumerate(lines, start=1):
|
|
118
|
+
if pattern_str.lower() in line.lower():
|
|
119
|
+
severity = self._str_to_severity(severity_str)
|
|
120
|
+
cwe = _CWE_MAPPING.get(pattern_name, "CWE-798")
|
|
121
|
+
findings.append(Finding(
|
|
122
|
+
concern=ConcernType.SECURITY,
|
|
123
|
+
severity=severity,
|
|
124
|
+
file=path,
|
|
125
|
+
line_start=i,
|
|
126
|
+
line_end=i,
|
|
127
|
+
summary=f"Potential {pattern_name.replace('_', ' ')} detected",
|
|
128
|
+
explanation=f"Pattern '{pattern_str}' found in {path}:{i}. "
|
|
129
|
+
f"This may be a hard-coded credential. {cwe}",
|
|
130
|
+
suggested_fix_diff=None,
|
|
131
|
+
confidence=self._compute_secret_confidence(pattern_name, line),
|
|
132
|
+
evidence=[EvidenceItem(
|
|
133
|
+
tool="secret_scanner",
|
|
134
|
+
result=cwe,
|
|
135
|
+
query=pattern_str,
|
|
136
|
+
)],
|
|
137
|
+
agent_version=self.AGENT_VERSION,
|
|
138
|
+
prompt_hash=self._prompt_hash("secret_scan", path, str(i)),
|
|
139
|
+
))
|
|
140
|
+
break
|
|
141
|
+
return findings
|
|
142
|
+
|
|
143
|
+
def _scan_diff_for_vulnerabilities(self, diff_files: list[dict]) -> list[Finding]:
|
|
144
|
+
"""Scan diff content for common vulnerability patterns."""
|
|
145
|
+
findings: list[Finding] = []
|
|
146
|
+
vuln_patterns: list[tuple[str, str, str, str]] = [
|
|
147
|
+
("sql_injection", "f\"SELECT", "high", "Potential SQL injection via f-string"),
|
|
148
|
+
("sql_injection2", "' + request", "high", "Potential SQL injection via string concat"),
|
|
149
|
+
("eval_usage", "eval(", "critical", "Use of eval() — potential code injection"),
|
|
150
|
+
("exec_usage", "exec(", "critical", "Use of exec() — potential code injection"),
|
|
151
|
+
("shell_injection", "subprocess.call.*shell=True", "critical", "Shell injection risk"),
|
|
152
|
+
("os_system", "os.system(", "high", "os.system() — potential command injection"),
|
|
153
|
+
("pickle_load", "pickle.load", "high", "Unsafe deserialization via pickle"),
|
|
154
|
+
("yaml_unsafe", "yaml.load(", "medium", "Unsafe YAML load — use yaml.safe_load"),
|
|
155
|
+
("path_traversal", "../", "medium", "Potential path traversal — validate input"),
|
|
156
|
+
("weak_hash", "md5(", "medium", "Weak hash function — use SHA-256 or stronger"),
|
|
157
|
+
("insecure_random", "random.randint", "low", "Cryptographically weak random"),
|
|
158
|
+
]
|
|
159
|
+
|
|
160
|
+
for file_info in diff_files:
|
|
161
|
+
path = file_info.get("path", "")
|
|
162
|
+
content = file_info.get("content", "")
|
|
163
|
+
additions = file_info.get("additions", "")
|
|
164
|
+
scan_text = additions if additions else content
|
|
165
|
+
|
|
166
|
+
for name, pattern, severity, explanation in vuln_patterns:
|
|
167
|
+
if pattern.lower() in scan_text.lower():
|
|
168
|
+
lines = scan_text.split("\n")
|
|
169
|
+
for i, line in enumerate(lines, start=1):
|
|
170
|
+
if pattern.lower() in line.lower():
|
|
171
|
+
findings.append(Finding(
|
|
172
|
+
concern=ConcernType.SECURITY,
|
|
173
|
+
severity=self._str_to_severity(severity),
|
|
174
|
+
file=path,
|
|
175
|
+
line_start=i,
|
|
176
|
+
line_end=i,
|
|
177
|
+
summary=f"{name.replace('_', ' ').title()} detected",
|
|
178
|
+
explanation=f"{explanation} at {path}:{i}",
|
|
179
|
+
suggested_fix_diff=self._suggested_fix(name),
|
|
180
|
+
confidence=0.75 if severity in ("high", "critical") else 0.55,
|
|
181
|
+
evidence=[EvidenceItem(
|
|
182
|
+
tool="static_analyzer",
|
|
183
|
+
result=name,
|
|
184
|
+
query=pattern,
|
|
185
|
+
)],
|
|
186
|
+
agent_version=self.AGENT_VERSION,
|
|
187
|
+
prompt_hash=self._prompt_hash("vuln_scan", path, str(i)),
|
|
188
|
+
))
|
|
189
|
+
break
|
|
190
|
+
return findings
|
|
191
|
+
|
|
192
|
+
# ── Semantic Security Search ──────────────────────────────────────
|
|
193
|
+
|
|
194
|
+
async def _semantic_security_search(
|
|
195
|
+
self,
|
|
196
|
+
*,
|
|
197
|
+
ctx: SpecialistContext,
|
|
198
|
+
queries: list[str],
|
|
199
|
+
semantic_index: SemanticIndex,
|
|
200
|
+
) -> list[Finding]:
|
|
201
|
+
findings: list[Finding] = []
|
|
202
|
+
diff_paths = {fi.get("path", "") for fi in ctx.diff_files}
|
|
203
|
+
|
|
204
|
+
for query in queries:
|
|
205
|
+
try:
|
|
206
|
+
results = await semantic_index.search_by_text(
|
|
207
|
+
f"{ctx.repo_owner}/{ctx.repo_name}", query, limit=3,
|
|
208
|
+
)
|
|
209
|
+
for chunk in results:
|
|
210
|
+
if chunk["file_path"] not in diff_paths:
|
|
211
|
+
continue
|
|
212
|
+
content = chunk["content"]
|
|
213
|
+
security_keywords = ["auth", "token", "password", "secret", "key", "session"]
|
|
214
|
+
if any(kw in content.lower() for kw in security_keywords):
|
|
215
|
+
findings.append(Finding(
|
|
216
|
+
concern=ConcernType.SECURITY,
|
|
217
|
+
severity=self._str_to_severity("medium"),
|
|
218
|
+
file=chunk["file_path"],
|
|
219
|
+
line_start=chunk["start_line"],
|
|
220
|
+
line_end=chunk["end_line"],
|
|
221
|
+
summary=f"Security-relevant code near search term: '{query}'",
|
|
222
|
+
explanation=(
|
|
223
|
+
f"File {chunk['file_path']} contains both '{query}' and "
|
|
224
|
+
f"security keywords. Review for proper handling."
|
|
225
|
+
),
|
|
226
|
+
suggested_fix_diff=None,
|
|
227
|
+
confidence=0.45,
|
|
228
|
+
evidence=[EvidenceItem(
|
|
229
|
+
tool="semantic_search",
|
|
230
|
+
query=query,
|
|
231
|
+
result=f"matched at {chunk['file_path']}:{chunk['start_line']}",
|
|
232
|
+
)],
|
|
233
|
+
agent_version=self.AGENT_VERSION,
|
|
234
|
+
prompt_hash=self._prompt_hash("semantic_search", query, chunk["file_path"]),
|
|
235
|
+
))
|
|
236
|
+
except Exception as exc:
|
|
237
|
+
logger.debug("Semantic search for '%s' failed: %s", query, exc)
|
|
238
|
+
|
|
239
|
+
return findings
|
|
240
|
+
|
|
241
|
+
# ── Confidence Computation (deterministic, per Orchestration doc §4) ─
|
|
242
|
+
|
|
243
|
+
def _compute_secret_confidence(self, pattern_name: str, line: str) -> float:
|
|
244
|
+
if pattern_name in ("PRIVATE_KEY",):
|
|
245
|
+
base_confidence = 0.95
|
|
246
|
+
elif pattern_name in ("AWS_ACCESS_KEY", "AWS_SECRET_KEY"):
|
|
247
|
+
base_confidence = 0.90
|
|
248
|
+
else:
|
|
249
|
+
base_confidence = 0.85
|
|
250
|
+
|
|
251
|
+
stripped = line.strip()
|
|
252
|
+
if stripped.startswith("#") or stripped.startswith("//") or stripped.startswith("*"):
|
|
253
|
+
base_confidence = max(0.1, base_confidence - 0.3)
|
|
254
|
+
|
|
255
|
+
return round(base_confidence, 2)
|
|
256
|
+
|
|
257
|
+
# ── Helpers ───────────────────────────────────────────────────────
|
|
258
|
+
|
|
259
|
+
@staticmethod
|
|
260
|
+
def _str_to_severity(s: str) -> "Severity":
|
|
261
|
+
from verdity.schemas import Severity
|
|
262
|
+
return Severity(s.lower())
|
|
263
|
+
|
|
264
|
+
@staticmethod
|
|
265
|
+
def _suggested_fix(vuln_name: str) -> str | None:
|
|
266
|
+
fixes: dict[str, str] = {
|
|
267
|
+
"sql_injection": "- cursor.execute(f\"SELECT ...\")\n+ cursor.execute(\"SELECT ... WHERE id = %s\", (user_id,))",
|
|
268
|
+
"eval_usage": "- result = eval(user_input)\n+ result = ast.literal_eval(user_input) # or a safer alternative",
|
|
269
|
+
"exec_usage": "- exec(user_code)\n+ # Avoid exec; use a sandboxed environment or pre-compiled functions",
|
|
270
|
+
"pickle_load": "- data = pickle.load(f)\n+ data = json.load(f) # or use a safe deserializer",
|
|
271
|
+
"yaml_unsafe": "- yaml.load(stream)\n+ yaml.safe_load(stream)",
|
|
272
|
+
"weak_hash": "- hashlib.md5(data)\n+ hashlib.sha256(data)",
|
|
273
|
+
}
|
|
274
|
+
return fixes.get(vuln_name)
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Testing Specialist Agent.
|
|
3
|
+
|
|
4
|
+
Reviews PR diffs for test coverage gaps and testing best practices.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import logging
|
|
10
|
+
|
|
11
|
+
from verdity.agents.base import BaseSpecialistAgent
|
|
12
|
+
from verdity.schemas import (
|
|
13
|
+
ConcernType,
|
|
14
|
+
EvidenceItem,
|
|
15
|
+
Finding,
|
|
16
|
+
SpecialistContext,
|
|
17
|
+
Severity,
|
|
18
|
+
)
|
|
19
|
+
from verdity.semantic_index import SemanticIndex
|
|
20
|
+
|
|
21
|
+
logger = logging.getLogger(__name__)
|
|
22
|
+
|
|
23
|
+
TEST_PATTERNS: list[tuple[str, str, str, str]] = [
|
|
24
|
+
("no_test_for_function", "def test_", "info", "Test function added — verify implementation coverage"),
|
|
25
|
+
("untested_branch", " pass", "medium", "Empty branch (pass) may indicate untested path"),
|
|
26
|
+
("mock_usage", "mock.patch", "info", "Mock usage detected — verify mock scope is appropriate"),
|
|
27
|
+
("assert_no_message", "assert ", "low", "Assertion found — verify test covers behavior not messages"),
|
|
28
|
+
]
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class TestingAgent(BaseSpecialistAgent):
|
|
32
|
+
"""Testing specialist agent for Verdity."""
|
|
33
|
+
|
|
34
|
+
AGENT_VERSION = "testing-agent@0.1.0"
|
|
35
|
+
SPECIALIST_NAME = "testing"
|
|
36
|
+
CONCERN_TYPE = ConcernType.TESTING
|
|
37
|
+
_input_tokens_per_finding = 200
|
|
38
|
+
_output_tokens_per_finding = 30
|
|
39
|
+
|
|
40
|
+
async def _scan(
|
|
41
|
+
self,
|
|
42
|
+
ctx: SpecialistContext,
|
|
43
|
+
semantic_index: SemanticIndex,
|
|
44
|
+
) -> list[Finding]:
|
|
45
|
+
findings: list[Finding] = []
|
|
46
|
+
|
|
47
|
+
for file_info in ctx.diff_files:
|
|
48
|
+
path = file_info.get("path", "")
|
|
49
|
+
additions = file_info.get("additions", "")
|
|
50
|
+
if not additions:
|
|
51
|
+
continue
|
|
52
|
+
|
|
53
|
+
for name, pattern, severity, explanation in TEST_PATTERNS:
|
|
54
|
+
if pattern.lower() in additions.lower():
|
|
55
|
+
lines = additions.split("\n")
|
|
56
|
+
for i, line in enumerate(lines, start=1):
|
|
57
|
+
if pattern.lower() in line.lower():
|
|
58
|
+
findings.append(Finding(
|
|
59
|
+
concern=ConcernType.TESTING,
|
|
60
|
+
severity=Severity(severity),
|
|
61
|
+
file=path,
|
|
62
|
+
line_start=i,
|
|
63
|
+
line_end=i,
|
|
64
|
+
summary=f"{name.replace('_', ' ').title()}",
|
|
65
|
+
explanation=f"{explanation} at {path}:{i}",
|
|
66
|
+
suggested_fix_diff=None,
|
|
67
|
+
confidence=0.5 if severity == "info" else 0.65,
|
|
68
|
+
evidence=[EvidenceItem(tool="test_analyzer", query=pattern, result=name)],
|
|
69
|
+
agent_version=self.AGENT_VERSION,
|
|
70
|
+
prompt_hash=self._prompt_hash(name, path, str(i)),
|
|
71
|
+
))
|
|
72
|
+
break
|
|
73
|
+
|
|
74
|
+
return findings
|