agentops-cockpit 0.9.5__py3-none-any.whl → 0.9.8__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agent_ops_cockpit/agent.py +44 -77
- agent_ops_cockpit/cache/semantic_cache.py +10 -21
- agent_ops_cockpit/cli/main.py +105 -153
- agent_ops_cockpit/eval/load_test.py +33 -50
- agent_ops_cockpit/eval/quality_climber.py +88 -93
- agent_ops_cockpit/eval/red_team.py +84 -25
- agent_ops_cockpit/mcp_server.py +26 -93
- agent_ops_cockpit/ops/arch_review.py +221 -147
- agent_ops_cockpit/ops/auditors/base.py +50 -0
- agent_ops_cockpit/ops/auditors/behavioral.py +31 -0
- agent_ops_cockpit/ops/auditors/compliance.py +35 -0
- agent_ops_cockpit/ops/auditors/dependency.py +48 -0
- agent_ops_cockpit/ops/auditors/finops.py +48 -0
- agent_ops_cockpit/ops/auditors/graph.py +49 -0
- agent_ops_cockpit/ops/auditors/pivot.py +51 -0
- agent_ops_cockpit/ops/auditors/reasoning.py +67 -0
- agent_ops_cockpit/ops/auditors/reliability.py +53 -0
- agent_ops_cockpit/ops/auditors/security.py +87 -0
- agent_ops_cockpit/ops/auditors/sme_v12.py +76 -0
- agent_ops_cockpit/ops/auditors/sovereignty.py +74 -0
- agent_ops_cockpit/ops/auditors/sre_a2a.py +179 -0
- agent_ops_cockpit/ops/benchmarker.py +97 -0
- agent_ops_cockpit/ops/cost_optimizer.py +15 -24
- agent_ops_cockpit/ops/discovery.py +214 -0
- agent_ops_cockpit/ops/evidence_bridge.py +30 -63
- agent_ops_cockpit/ops/frameworks.py +124 -1
- agent_ops_cockpit/ops/git_portal.py +74 -0
- agent_ops_cockpit/ops/mcp_hub.py +19 -42
- agent_ops_cockpit/ops/orchestrator.py +477 -277
- agent_ops_cockpit/ops/policy_engine.py +38 -38
- agent_ops_cockpit/ops/reliability.py +121 -52
- agent_ops_cockpit/ops/remediator.py +54 -0
- agent_ops_cockpit/ops/secret_scanner.py +34 -22
- agent_ops_cockpit/ops/swarm.py +17 -27
- agent_ops_cockpit/ops/ui_auditor.py +67 -6
- agent_ops_cockpit/ops/watcher.py +41 -70
- agent_ops_cockpit/ops/watchlist.json +30 -0
- agent_ops_cockpit/optimizer.py +161 -384
- agent_ops_cockpit/tests/test_arch_review.py +6 -6
- agent_ops_cockpit/tests/test_discovery.py +96 -0
- agent_ops_cockpit/tests/test_ops_core.py +56 -0
- agent_ops_cockpit/tests/test_orchestrator_fleet.py +73 -0
- agent_ops_cockpit/tests/test_persona_architect.py +75 -0
- agent_ops_cockpit/tests/test_persona_finops.py +31 -0
- agent_ops_cockpit/tests/test_persona_security.py +55 -0
- agent_ops_cockpit/tests/test_persona_sre.py +43 -0
- agent_ops_cockpit/tests/test_persona_ux.py +42 -0
- agent_ops_cockpit/tests/test_quality_climber.py +2 -2
- agent_ops_cockpit/tests/test_remediator.py +75 -0
- agent_ops_cockpit/tests/test_ui_auditor.py +52 -0
- agentops_cockpit-0.9.8.dist-info/METADATA +172 -0
- agentops_cockpit-0.9.8.dist-info/RECORD +71 -0
- agent_ops_cockpit/tests/test_optimizer.py +0 -68
- agent_ops_cockpit/tests/test_red_team.py +0 -35
- agent_ops_cockpit/tests/test_secret_scanner.py +0 -24
- agentops_cockpit-0.9.5.dist-info/METADATA +0 -246
- agentops_cockpit-0.9.5.dist-info/RECORD +0 -47
- {agentops_cockpit-0.9.5.dist-info → agentops_cockpit-0.9.8.dist-info}/WHEEL +0 -0
- {agentops_cockpit-0.9.5.dist-info → agentops_cockpit-0.9.8.dist-info}/entry_points.txt +0 -0
- {agentops_cockpit-0.9.5.dist-info → agentops_cockpit-0.9.8.dist-info}/licenses/LICENSE +0 -0
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
from tenacity import retry, wait_exponential, stop_after_attempt
|
|
2
|
+
from tenacity import retry, wait_exponential, stop_after_attempt
|
|
3
|
+
from tenacity import retry, wait_exponential, stop_after_attempt
|
|
4
|
+
import ast
|
|
5
|
+
import json
|
|
6
|
+
import os
|
|
7
|
+
from typing import List
|
|
8
|
+
from .base import BaseAuditor, AuditFinding
|
|
9
|
+
|
|
10
|
+
class BehavioralAuditor(BaseAuditor):
|
|
11
|
+
"""
|
|
12
|
+
v1.1 Enterprise Architect: Behavioral & Trace Auditor.
|
|
13
|
+
Compares runtime traces (JSON) against static code promises.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
def audit(self, tree: ast.AST, content: str, file_path: str) -> List[AuditFinding]:
|
|
17
|
+
findings = []
|
|
18
|
+
trace_path = os.path.join(os.path.dirname(file_path), 'trace.json')
|
|
19
|
+
if not os.path.exists(trace_path):
|
|
20
|
+
return []
|
|
21
|
+
try:
|
|
22
|
+
with open(trace_path, 'r') as f:
|
|
23
|
+
trace_data = json.load(f)
|
|
24
|
+
if 'mask' in content.lower() or 'pii' in content.lower():
|
|
25
|
+
for entry in trace_data.get('logs', []):
|
|
26
|
+
message = entry.get('message', '')
|
|
27
|
+
if '@' in message and '.' in message:
|
|
28
|
+
findings.append(AuditFinding(category='🎭 Behavioral', title='Trace-to-Code Mismatch (PII Leak)', description=f"Code promises PII masking, but trace.json contains raw email patterns at {entry.get('timestamp')}.", impact='CRITICAL', roi="Ensure semantic masking logic handles 'suffix+alias' patterns correctly.", file_path=file_path))
|
|
29
|
+
except Exception:
|
|
30
|
+
pass
|
|
31
|
+
return findings
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import ast
|
|
2
|
+
from typing import List
|
|
3
|
+
from .base import BaseAuditor, AuditFinding
|
|
4
|
+
|
|
5
|
+
class ComplianceAuditor(BaseAuditor):
|
|
6
|
+
"""
|
|
7
|
+
v1.1 Enterprise Architect: Governance & Compliance Auditor.
|
|
8
|
+
Maps code flaws to specific audit controls (SOC2, HIPAA, NIST).
|
|
9
|
+
"""
|
|
10
|
+
def audit(self, tree: ast.AST, content: str, file_path: str) -> List[AuditFinding]:
|
|
11
|
+
findings = []
|
|
12
|
+
|
|
13
|
+
# Check for Audit Logging (SOC2 CC6.1 - Access Monitoring)
|
|
14
|
+
if "log" not in content.lower():
|
|
15
|
+
findings.append(AuditFinding(
|
|
16
|
+
category="⚖️ Compliance",
|
|
17
|
+
title="SOC2 Control Gap: Missing Transit Logging",
|
|
18
|
+
description="No logging detected in mission-critical file. SOC2 CC6.1 requires audit trails for all system access.",
|
|
19
|
+
impact="HIGH",
|
|
20
|
+
roi="Critical for passing external audits and root-cause analysis.",
|
|
21
|
+
file_path=file_path
|
|
22
|
+
))
|
|
23
|
+
|
|
24
|
+
# Check for HIPAA (Encryption at Rest - Simplified)
|
|
25
|
+
if "sql" in content.lower() or "db" in content.lower():
|
|
26
|
+
if "encrypt" not in content.lower() and "secret" not in content.lower():
|
|
27
|
+
findings.append(AuditFinding(
|
|
28
|
+
category="⚖️ Compliance",
|
|
29
|
+
title="HIPAA Risk: Potential Unencrypted ePHI",
|
|
30
|
+
description="Database interaction detected without explicit encryption or secret management headers.",
|
|
31
|
+
impact="CRITICAL",
|
|
32
|
+
roi="Avoid legal penalties by enforcing encryption headers in database client configuration."
|
|
33
|
+
))
|
|
34
|
+
|
|
35
|
+
return findings
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
import ast
|
|
2
|
+
import os
|
|
3
|
+
import re
|
|
4
|
+
from typing import List
|
|
5
|
+
from .base import BaseAuditor, AuditFinding
|
|
6
|
+
|
|
7
|
+
class DependencyAuditor(BaseAuditor):
|
|
8
|
+
"""
|
|
9
|
+
v1.1 Enterprise Architect: Dependency Fragility Auditor.
|
|
10
|
+
Scans pyproject.toml and requirements.txt for version drift and known conflicts.
|
|
11
|
+
"""
|
|
12
|
+
KNOW_CONFLICTS = [
|
|
13
|
+
("langchain", "0.1.0", "crewai", "0.30.0", "Breaking change in BaseCallbackHandler. Expect runtime crashes during tool execution.")
|
|
14
|
+
]
|
|
15
|
+
|
|
16
|
+
def audit(self, tree: ast.AST, content: str, file_path: str) -> List[AuditFinding]:
|
|
17
|
+
findings = []
|
|
18
|
+
|
|
19
|
+
# Only check dependencies once per audit root (if we detect we are in the root)
|
|
20
|
+
if not file_path.endswith(("pyproject.toml", "requirements.txt")):
|
|
21
|
+
return []
|
|
22
|
+
|
|
23
|
+
# Simple regex for dependency parsing (v1.1)
|
|
24
|
+
deps = re.findall(r"(['\"]?\w+['\"]?)\s*[>=<]{1,2}\s*([\d\.]*)", content)
|
|
25
|
+
dep_map = {name.strip("'\""): version for name, version in deps}
|
|
26
|
+
|
|
27
|
+
for p1, v1_min, p2, v2_min, reason in self.KNOW_CONFLICTS:
|
|
28
|
+
if p1 in dep_map and p2 in dep_map:
|
|
29
|
+
findings.append(AuditFinding(
|
|
30
|
+
category="📦 Dependency",
|
|
31
|
+
title="Version Drift Conflict Detected",
|
|
32
|
+
description=f"Detected potential conflict between {p1} and {p2}. {reason}",
|
|
33
|
+
impact="HIGH",
|
|
34
|
+
roi="Prevent runtime failures and dependency hell before deployment.",
|
|
35
|
+
file_path=file_path
|
|
36
|
+
))
|
|
37
|
+
|
|
38
|
+
# Licensing Check
|
|
39
|
+
if "agpl" in content.lower():
|
|
40
|
+
findings.append(AuditFinding(
|
|
41
|
+
category="⚖️ Compliance",
|
|
42
|
+
title="Restrictive License Warning",
|
|
43
|
+
description="AGPL-licensed dependency found. This may require full source disclosure for the entire project.",
|
|
44
|
+
impact="MEDIUM",
|
|
45
|
+
roi="Avoid legal risk by switching to MIT/Apache alternatives."
|
|
46
|
+
))
|
|
47
|
+
|
|
48
|
+
return findings
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
from tenacity import retry, wait_exponential, stop_after_attempt
|
|
2
|
+
import ast
|
|
3
|
+
import re
|
|
4
|
+
from typing import List
|
|
5
|
+
from .base import BaseAuditor, AuditFinding
|
|
6
|
+
|
|
7
|
+
class FinOpsAuditor(BaseAuditor):
|
|
8
|
+
"""
|
|
9
|
+
v1.2 Principal SME: FinOps & Economic Sustainability Auditor.
|
|
10
|
+
Projects TCO and identifies missed caching opportunities for high-volume agents.
|
|
11
|
+
"""
|
|
12
|
+
MODEL_PRICES = {
|
|
13
|
+
'gemini-1.5-pro': 3.5,
|
|
14
|
+
'gemini-1.5-flash': 0.35,
|
|
15
|
+
'gpt-4': 10.0,
|
|
16
|
+
'gpt-3.5': 0.5
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
def audit(self, tree: ast.AST, content: str, file_path: str) -> List[AuditFinding]:
|
|
20
|
+
findings = []
|
|
21
|
+
|
|
22
|
+
# 1. Model TCO Projection
|
|
23
|
+
for model, price in self.MODEL_PRICES.items():
|
|
24
|
+
if model in content.lower():
|
|
25
|
+
has_loop = re.search(r'for\s+\w+\s+in', content)
|
|
26
|
+
multiplier = 10 if has_loop else 1
|
|
27
|
+
projected_cost = price * multiplier
|
|
28
|
+
findings.append(AuditFinding(
|
|
29
|
+
category="💰 FinOps",
|
|
30
|
+
title=f"Inference Cost Projection ({model})",
|
|
31
|
+
description=f"Detected {model} usage. Projected TCO over 1M tokens: ${projected_cost:.2f}.",
|
|
32
|
+
impact="INFO",
|
|
33
|
+
roi=f"Switching to Flash-equivalent could reduce projected cost to ${0.35 * multiplier:.2f}."
|
|
34
|
+
))
|
|
35
|
+
|
|
36
|
+
# 2. Context Caching Opportunity
|
|
37
|
+
docstrings = re.findall(r'"""([\s\S]*?)"""|\'\'\'([\s\S]*?)\'\'\'', content)
|
|
38
|
+
has_large_prompt = any(len(d[0] or d[1]) > 500 for d in docstrings)
|
|
39
|
+
if has_large_prompt and 'CachingConfig' not in content:
|
|
40
|
+
findings.append(AuditFinding(
|
|
41
|
+
category="💰 FinOps",
|
|
42
|
+
title="Context Caching Opportunity",
|
|
43
|
+
description="Large static system instructions detected without CachingConfig.",
|
|
44
|
+
impact="HIGH",
|
|
45
|
+
roi="Implement Vertex AI Context Caching to reduce repeated prefix costs by 90%."
|
|
46
|
+
))
|
|
47
|
+
|
|
48
|
+
return findings
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import ast
|
|
2
|
+
from typing import List
|
|
3
|
+
from .base import BaseAuditor, AuditFinding
|
|
4
|
+
|
|
5
|
+
class DeepGraphAuditor(BaseAuditor):
|
|
6
|
+
"""
|
|
7
|
+
Analyzes the 'Seams' between components and identifies dependency risks.
|
|
8
|
+
"""
|
|
9
|
+
def audit(self, tree: ast.AST, content: str, file_path: str) -> List[AuditFinding]:
|
|
10
|
+
findings = []
|
|
11
|
+
|
|
12
|
+
# 1. Dependency Cycle Detection (Simplified for AST)
|
|
13
|
+
# Look for circular imports or cross-module logic smells
|
|
14
|
+
imports = []
|
|
15
|
+
for node in ast.walk(tree):
|
|
16
|
+
if isinstance(node, (ast.Import, ast.ImportFrom)):
|
|
17
|
+
if isinstance(node, ast.Import):
|
|
18
|
+
for alias in node.names:
|
|
19
|
+
imports.append(alias.name)
|
|
20
|
+
else:
|
|
21
|
+
if node.module:
|
|
22
|
+
imports.append(node.module)
|
|
23
|
+
|
|
24
|
+
# 2. Hanging Tool Call Detection (Zombies)
|
|
25
|
+
# Look for async calls without timeouts or cancellation guards
|
|
26
|
+
for node in ast.walk(tree):
|
|
27
|
+
if isinstance(node, ast.Await):
|
|
28
|
+
if isinstance(node.value, ast.Call):
|
|
29
|
+
# Check for 'timeout' or 'wait' arguments in the call
|
|
30
|
+
has_timeout = any(kw.arg == "timeout" for kw in node.value.keywords)
|
|
31
|
+
if not has_timeout:
|
|
32
|
+
func_name = ""
|
|
33
|
+
if isinstance(node.value.func, ast.Name):
|
|
34
|
+
func_name = node.value.func.id
|
|
35
|
+
elif isinstance(node.value.func, ast.Attribute):
|
|
36
|
+
func_name = node.value.func.attr
|
|
37
|
+
|
|
38
|
+
if any(x in func_name.lower() for x in ["get", "post", "fetch", "invoke", "call"]):
|
|
39
|
+
findings.append(AuditFinding(
|
|
40
|
+
category="🧗 Resiliency",
|
|
41
|
+
title="Zombie Tool Call Detected",
|
|
42
|
+
description=f"Async call to '{func_name}' lacks a timeout. This risks 'Hanging Workers' in high-concurrency environments.",
|
|
43
|
+
impact="HIGH",
|
|
44
|
+
roi="Prevents thread-pool exhaustion and 99th percentile latency spikes.",
|
|
45
|
+
line_number=node.lineno,
|
|
46
|
+
file_path=file_path
|
|
47
|
+
))
|
|
48
|
+
|
|
49
|
+
return findings
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import ast
|
|
2
|
+
import re
|
|
3
|
+
from typing import List
|
|
4
|
+
from .base import BaseAuditor, AuditFinding
|
|
5
|
+
|
|
6
|
+
class PivotAuditor(BaseAuditor):
|
|
7
|
+
"""
|
|
8
|
+
v1.2 Principal SME: Strategic Pivot Auditor.
|
|
9
|
+
Reasons across multiple dimensions (Cost, Sovereignty, Protocol) to suggest
|
|
10
|
+
high-level architectural shifts (e.g., OpenAI -> Gemma2, REST -> MCP).
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
def audit(self, tree: ast.AST, content: str, file_path: str) -> List[AuditFinding]:
|
|
14
|
+
findings = []
|
|
15
|
+
|
|
16
|
+
# 1. Model Pivot: OpenAI/GPT -> Sovereign/Open Source (Gemma2)
|
|
17
|
+
if re.search(r"gpt-4|gpt-3\.5|openai", content.lower()):
|
|
18
|
+
findings.append(AuditFinding(
|
|
19
|
+
category="🚀 Strategic Pivot",
|
|
20
|
+
title="Sovereign Model Migration Opportunity",
|
|
21
|
+
description="Detected OpenAI dependency. For maximum Data Sovereignty and 40% TCO reduction, consider pivoting to Gemma2 or Llama3-70B on Vertex AI Prediction endpoints.",
|
|
22
|
+
impact="HIGH",
|
|
23
|
+
roi="Eliminates cross-border data risk and reduces projected inference TCO.",
|
|
24
|
+
file_path=file_path
|
|
25
|
+
))
|
|
26
|
+
|
|
27
|
+
# 2. Protocol Pivot: Legacy REST -> Model Context Protocol (MCP)
|
|
28
|
+
# Look for raw requests or aiohttp calls within 'tools' or 'mcp' directories
|
|
29
|
+
if ("tools" in file_path or "mcp" in file_path) and re.search(r"requests\.|aiohttp\.", content):
|
|
30
|
+
findings.append(AuditFinding(
|
|
31
|
+
category="🚀 Strategic Pivot",
|
|
32
|
+
title="Legacy Tooling -> MCP Migration",
|
|
33
|
+
description="Detected raw REST calls in a tool-heavy module. Migrating to Model Context Protocol (MCP) would provide unified governance, better tool discovery, and standardized error handling.",
|
|
34
|
+
impact="MEDIUM",
|
|
35
|
+
roi="Future-proofs the tool layer and enables interoperability with MCP-native ecosystems.",
|
|
36
|
+
file_path=file_path
|
|
37
|
+
))
|
|
38
|
+
|
|
39
|
+
# 3. Compute Pivot: Cloud Run -> GKE (Scale reasoning)
|
|
40
|
+
# Heuristic: If we see many service definitions or complex scaling params
|
|
41
|
+
if "deploy" in content.lower() and "scaling" in content.lower():
|
|
42
|
+
findings.append(AuditFinding(
|
|
43
|
+
category="🚀 Strategic Pivot",
|
|
44
|
+
title="Compute Scaling Optimization",
|
|
45
|
+
description="Detected complex scaling logic. If traffic exceeds 10k RPS, consider pivoting from Cloud Run to GKE with Anthos for hybrid-cloud sovereignty.",
|
|
46
|
+
impact="INFO",
|
|
47
|
+
roi="Optimizes unit cost at extreme scale while maintaining multi-cloud flexibility.",
|
|
48
|
+
file_path=file_path
|
|
49
|
+
))
|
|
50
|
+
|
|
51
|
+
return findings
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import ast
|
|
2
|
+
import re
|
|
3
|
+
from typing import List
|
|
4
|
+
from .base import BaseAuditor, AuditFinding
|
|
5
|
+
|
|
6
|
+
class ReasoningAuditor(BaseAuditor):
|
|
7
|
+
"""
|
|
8
|
+
The 'Shadow Consultant' reasoning engine.
|
|
9
|
+
Performs Phase 2 Active Reasoning: identifying trade-offs, bottlenecks, and intent.
|
|
10
|
+
"""
|
|
11
|
+
def audit(self, tree: ast.AST, content: str, file_path: str) -> List[AuditFinding]:
|
|
12
|
+
findings = []
|
|
13
|
+
|
|
14
|
+
# 1. Intent Detection: Look for 'Seams' between frameworks
|
|
15
|
+
if "langgraph" in content.lower() and "crewai" in content.lower():
|
|
16
|
+
findings.append(AuditFinding(
|
|
17
|
+
category="🏗️ Architecture",
|
|
18
|
+
title="Strategic Conflict: Multi-Orchestrator Setup",
|
|
19
|
+
description="Detected both LangGraph and CrewAI. Using two loop managers is a 'High-Entropy' pattern that often leads to cyclic state deadlocks.",
|
|
20
|
+
impact="HIGH",
|
|
21
|
+
roi="Recommend using LangGraph for 'Brain' and CrewAI for 'Task Workers' to ensure state consistency.",
|
|
22
|
+
file_path=file_path
|
|
23
|
+
))
|
|
24
|
+
|
|
25
|
+
# 2. Logic Probing: Sequential await profiling
|
|
26
|
+
async_funcs = [n for n in ast.walk(tree) if isinstance(n, ast.AsyncFunctionDef)]
|
|
27
|
+
for func in async_funcs:
|
|
28
|
+
awaits = [n for n in ast.walk(func) if isinstance(n, ast.Await)]
|
|
29
|
+
if len(awaits) >= 3:
|
|
30
|
+
# Check if awaits are independent (not using results of previous yields)
|
|
31
|
+
# This is a 'Reasoning' step over the AST
|
|
32
|
+
findings.append(AuditFinding(
|
|
33
|
+
category="🧗 Reliability",
|
|
34
|
+
title="Sequential Data Fetching Bottleneck",
|
|
35
|
+
description=f"Function '{func.name}' has {len(awaits)} sequential await calls. This increases latency lineary (T1+T2+T3).",
|
|
36
|
+
impact="MEDIUM",
|
|
37
|
+
roi="Parallelizing these calls could reduce latency by up to 60%.",
|
|
38
|
+
line_number=func.lineno,
|
|
39
|
+
file_path=file_path
|
|
40
|
+
))
|
|
41
|
+
|
|
42
|
+
# 3. FinOps Reasoning: High-tier model in loops
|
|
43
|
+
# (Simulated reasoning: we look for model strings and for loops together)
|
|
44
|
+
if re.search(r"gemini-1\.5-pro|gpt-4", content) and re.search(r"for\s+\w+\s+in", content):
|
|
45
|
+
if any(x in content.lower() for x in ["classify", "label", "is_", "has_"]):
|
|
46
|
+
findings.append(AuditFinding(
|
|
47
|
+
category="💰 FinOps",
|
|
48
|
+
title="Model Efficiency Regression",
|
|
49
|
+
description="High-tier model (Pro/GPT-4) detected inside a loop performing simple classification tasks.",
|
|
50
|
+
impact="HIGH",
|
|
51
|
+
roi="Pivoting to Gemini 1.5 Flash for this loop reduces token spend by 90% with zero accuracy loss.",
|
|
52
|
+
file_path=file_path
|
|
53
|
+
))
|
|
54
|
+
|
|
55
|
+
# 4. Context Bloat Analysis (Prompt Logic)
|
|
56
|
+
long_prompts = re.findall(r"['\"]{3}([\s\S]{5000,})['\"]{3}", content)
|
|
57
|
+
if long_prompts:
|
|
58
|
+
findings.append(AuditFinding(
|
|
59
|
+
category="📉 Efficiency",
|
|
60
|
+
title="Architectural Prompt Bloat",
|
|
61
|
+
description="Massive static context (>5k chars) detected in system instruction. This risks 'Lost in the Middle' hallucinations.",
|
|
62
|
+
impact="MEDIUM",
|
|
63
|
+
roi="Pivot to a RAG (Retrieval Augmented Generation) pattern to improve factual grounding accuracy.",
|
|
64
|
+
file_path=file_path
|
|
65
|
+
))
|
|
66
|
+
|
|
67
|
+
return findings
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
from tenacity import retry, wait_exponential, stop_after_attempt
|
|
2
|
+
import ast
|
|
3
|
+
from typing import List
|
|
4
|
+
from .base import BaseAuditor, AuditFinding
|
|
5
|
+
|
|
6
|
+
class ReliabilityAuditor(BaseAuditor):
|
|
7
|
+
"""
|
|
8
|
+
Analyzes code for execution patterns, identifying sequential bottlenecks and missing resiliency.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
def audit(self, tree: ast.AST, content: str, file_path: str) -> List[AuditFinding]:
|
|
12
|
+
findings = []
|
|
13
|
+
for node in ast.walk(tree):
|
|
14
|
+
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
|
15
|
+
await_nodes = [n for n in ast.walk(node) if isinstance(n, ast.Await)]
|
|
16
|
+
if len(await_nodes) > 2:
|
|
17
|
+
findings.append(AuditFinding(category='🧗 Reliability & Perf', title='Sequential Bottleneck Detected', description="Multiple sequential 'await' calls identified. This increases total latency linearly.", impact='MEDIUM', roi='Reduces latency by up to 50% using asyncio.gather().', line_number=node.lineno, file_path=file_path))
|
|
18
|
+
if isinstance(node, ast.Call):
|
|
19
|
+
func_name = ''
|
|
20
|
+
if isinstance(node.func, ast.Name):
|
|
21
|
+
func_name = node.func.id
|
|
22
|
+
elif isinstance(node.func, ast.Attribute):
|
|
23
|
+
func_name = node.func.attr
|
|
24
|
+
if any((x in func_name.lower() for x in ['get', 'post', 'request', 'fetch', 'query'])):
|
|
25
|
+
parent = self._get_parent_function(tree, node)
|
|
26
|
+
if parent and (not self._is_decorated_with_retry(parent)):
|
|
27
|
+
findings.append(AuditFinding(category='🧗 Reliability', title='Missing Resiliency Logic', description=f"External call '{func_name}' is not protected by retry logic.", impact='HIGH', roi='Increases up-time and handles transient network failures.', line_number=node.lineno, file_path=file_path))
|
|
28
|
+
if isinstance(node, (ast.Assign, ast.AnnAssign)):
|
|
29
|
+
if isinstance(node.value, (ast.Constant, ast.JoinedStr)):
|
|
30
|
+
val = ''
|
|
31
|
+
if isinstance(node.value, ast.Constant) and isinstance(node.value.value, str):
|
|
32
|
+
val = node.value.value
|
|
33
|
+
elif isinstance(node.value, ast.JoinedStr):
|
|
34
|
+
val = ''.join([s.value for s in node.value.values if isinstance(s, ast.Constant)])
|
|
35
|
+
if len(val) > 20 and any((x in val.lower() for x in ['act as', 'you are', 'instruction'])):
|
|
36
|
+
if not any((x in val.lower() for x in ["don't know", 'unsure', 'refuse', 'do not make up'])):
|
|
37
|
+
findings.append(AuditFinding(category='🧗 Reliability', title='High Hallucination Risk', description="System prompt lacks negative constraints (e.g., 'If you don't know, say I don't know').", impact='HIGH', roi='Reduces autonomous failures by enforcing refusal boundaries.', line_number=node.lineno, file_path=file_path))
|
|
38
|
+
return findings
|
|
39
|
+
|
|
40
|
+
def _get_parent_function(self, tree, node):
|
|
41
|
+
for p in ast.walk(tree):
|
|
42
|
+
if isinstance(p, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
|
43
|
+
if any((n is node for n in ast.walk(p))):
|
|
44
|
+
return p
|
|
45
|
+
return None
|
|
46
|
+
|
|
47
|
+
def _is_decorated_with_retry(self, node):
|
|
48
|
+
for decorator in node.decorator_list:
|
|
49
|
+
if isinstance(decorator, ast.Name) and 'retry' in decorator.id.lower():
|
|
50
|
+
return True
|
|
51
|
+
if isinstance(decorator, ast.Call) and isinstance(decorator.func, ast.Name) and ('retry' in decorator.func.id.lower()):
|
|
52
|
+
return True
|
|
53
|
+
return False
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
import ast
|
|
2
|
+
import re
|
|
3
|
+
from typing import List
|
|
4
|
+
from .base import BaseAuditor, AuditFinding
|
|
5
|
+
|
|
6
|
+
class SecurityAuditor(BaseAuditor):
|
|
7
|
+
"""
|
|
8
|
+
Reasoning-based Security Auditor.
|
|
9
|
+
Scans for 'Seams' where unsanitized user input flows into LLM calls or Tool execution.
|
|
10
|
+
"""
|
|
11
|
+
def audit(self, tree: ast.AST, content: str, file_path: str) -> List[AuditFinding]:
|
|
12
|
+
findings = []
|
|
13
|
+
|
|
14
|
+
# Track variables that might contain raw user input
|
|
15
|
+
input_vars = set(["prompt", "user_input", "query", "request"])
|
|
16
|
+
sanitized_vars = set()
|
|
17
|
+
|
|
18
|
+
for node in ast.walk(tree):
|
|
19
|
+
# Track sanitization calls
|
|
20
|
+
if isinstance(node, ast.Assign):
|
|
21
|
+
if isinstance(node.value, ast.Call):
|
|
22
|
+
# If we call a scrubber, mark the target as sanitized
|
|
23
|
+
func_name = ""
|
|
24
|
+
if isinstance(node.value.func, ast.Name):
|
|
25
|
+
func_name = node.value.func.id
|
|
26
|
+
elif isinstance(node.value.func, ast.Attribute):
|
|
27
|
+
func_name = node.value.func.attr
|
|
28
|
+
|
|
29
|
+
if any(x in func_name.lower() for x in ["scrub", "mask", "sanitize", "guard", "filter"]):
|
|
30
|
+
for target in node.targets:
|
|
31
|
+
if isinstance(target, ast.Name):
|
|
32
|
+
sanitized_vars.add(target.id)
|
|
33
|
+
|
|
34
|
+
# Detect LLM invocations with unsanitized inputs
|
|
35
|
+
if isinstance(node, ast.Call):
|
|
36
|
+
func_name = ""
|
|
37
|
+
if isinstance(node.func, ast.Name):
|
|
38
|
+
func_name = node.func.id
|
|
39
|
+
elif isinstance(node.func, ast.Attribute):
|
|
40
|
+
func_name = node.func.attr
|
|
41
|
+
|
|
42
|
+
if any(x in func_name.lower() for x in ["invoke", "generate", "call", "chat", "predict"]):
|
|
43
|
+
# Check arguments
|
|
44
|
+
for arg in node.args:
|
|
45
|
+
if isinstance(arg, ast.Name) and (arg.id in input_vars) and (arg.id not in sanitized_vars):
|
|
46
|
+
findings.append(AuditFinding(
|
|
47
|
+
category="🛡️ Security",
|
|
48
|
+
title="Prompt Injection Susceptibility",
|
|
49
|
+
description=f"The variable '{arg.id}' flows into an LLM call without detected sanitization logic (e.g., scrub/guard).",
|
|
50
|
+
impact="CRITICAL",
|
|
51
|
+
roi="Prevents prompt injection attacks by 99%.",
|
|
52
|
+
line_number=node.lineno,
|
|
53
|
+
file_path=file_path
|
|
54
|
+
))
|
|
55
|
+
|
|
56
|
+
# Secret Scanning (v1.1)
|
|
57
|
+
for node in ast.walk(tree):
|
|
58
|
+
if isinstance(node, ast.Assign):
|
|
59
|
+
for target in node.targets:
|
|
60
|
+
if isinstance(target, ast.Name):
|
|
61
|
+
if any(x in target.id.lower() for x in ["key", "token", "secret", "password", "auth"]):
|
|
62
|
+
if isinstance(node.value, ast.Constant) and isinstance(node.value.value, str):
|
|
63
|
+
val = node.value.value
|
|
64
|
+
if len(val) > 16 and re.search(r"\d", val) and re.search(r"[a-zA-Z]", val):
|
|
65
|
+
findings.append(AuditFinding(
|
|
66
|
+
category="🛡️ Security",
|
|
67
|
+
title="Hardcoded Secret Detected",
|
|
68
|
+
description=f"Variable '{target.id}' appears to contain a hardcoded credential.",
|
|
69
|
+
impact="CRITICAL",
|
|
70
|
+
roi="Prevent catastrophic credential leaks by using Google Secret Manager.",
|
|
71
|
+
line_number=node.lineno,
|
|
72
|
+
file_path=file_path
|
|
73
|
+
))
|
|
74
|
+
|
|
75
|
+
# Look for TODOs in comments
|
|
76
|
+
if "// todo" in content.lower() or "# todo" in content.lower():
|
|
77
|
+
if "mask" in content.lower() or "pii" in content.lower():
|
|
78
|
+
findings.append(AuditFinding(
|
|
79
|
+
category="🛡️ Security",
|
|
80
|
+
title="Incomplete PII Protection",
|
|
81
|
+
description="Source code contains 'TODO' comments related to PII masking. Active protection is currently absent.",
|
|
82
|
+
impact="HIGH",
|
|
83
|
+
roi="Closes compliance gap for GDPR/SOC2.",
|
|
84
|
+
file_path=file_path
|
|
85
|
+
))
|
|
86
|
+
|
|
87
|
+
return findings
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
import ast
|
|
2
|
+
import re
|
|
3
|
+
from typing import List
|
|
4
|
+
from .base import BaseAuditor, AuditFinding
|
|
5
|
+
|
|
6
|
+
class HITLAuditor(BaseAuditor):
|
|
7
|
+
"""
|
|
8
|
+
v1.2 Principal SME: Human-in-the-Loop (HITL) Policy Auditor.
|
|
9
|
+
Flags high-risk "write" permissions that lack explicit approval gates.
|
|
10
|
+
"""
|
|
11
|
+
SENSITIVE_ACTIONS = [
|
|
12
|
+
(r"(?i)transfer", "Financial Transfer"),
|
|
13
|
+
(r"(?i)delete", "Resource Deletion"),
|
|
14
|
+
(r"(?i)update_policy", "Policy Modification"),
|
|
15
|
+
(r"(?i)send_email", "External Communication"),
|
|
16
|
+
(r"(?i)purchase", "Procurement")
|
|
17
|
+
]
|
|
18
|
+
|
|
19
|
+
def audit(self, tree: ast.AST, content: str, file_path: str) -> List[AuditFinding]:
|
|
20
|
+
findings = []
|
|
21
|
+
|
|
22
|
+
for node in ast.walk(tree):
|
|
23
|
+
# Check function definitions for sensitive names
|
|
24
|
+
if isinstance(node, ast.FunctionDef):
|
|
25
|
+
for pattern, category in self.SENSITIVE_ACTIONS:
|
|
26
|
+
if re.search(pattern, node.name):
|
|
27
|
+
# Look for 'human_approval' or 'require_approval' in arguments or decorator
|
|
28
|
+
has_gate = any(arg.arg in ["human_approval", "require_approval", "gate"] for arg in node.args.args)
|
|
29
|
+
|
|
30
|
+
# Also check decorators
|
|
31
|
+
if not has_gate:
|
|
32
|
+
for decorator in node.decorator_list:
|
|
33
|
+
if isinstance(decorator, ast.Name) and "gate" in decorator.id.lower():
|
|
34
|
+
has_gate = True
|
|
35
|
+
elif isinstance(decorator, ast.Call) and isinstance(decorator.func, ast.Name) and "gate" in decorator.func.id.lower():
|
|
36
|
+
has_gate = True
|
|
37
|
+
|
|
38
|
+
if not has_gate:
|
|
39
|
+
findings.append(AuditFinding(
|
|
40
|
+
category="🛡️ HITL Guardrail",
|
|
41
|
+
title=f"Ungated {category} Action",
|
|
42
|
+
description=f"Function '{node.name}' performs a high-risk action but lacks a 'human_approval' flag or security gate.",
|
|
43
|
+
impact="CRITICAL",
|
|
44
|
+
roi="Prevents autonomous catastrophic failures and unauthorized financial moves.",
|
|
45
|
+
line_number=node.lineno,
|
|
46
|
+
file_path=file_path
|
|
47
|
+
))
|
|
48
|
+
|
|
49
|
+
return findings
|
|
50
|
+
|
|
51
|
+
class A2AAuditor(BaseAuditor):
|
|
52
|
+
"""
|
|
53
|
+
v1.2 Principal SME: Agent-to-Agent (A2A) Efficiency Auditor.
|
|
54
|
+
Detects "Chatter Bloat" where agents pass excessive state objects.
|
|
55
|
+
"""
|
|
56
|
+
def audit(self, tree: ast.AST, content: str, file_path: str) -> List[AuditFinding]:
|
|
57
|
+
findings = []
|
|
58
|
+
|
|
59
|
+
# Look for patterns like passing 'state' or 'full_context' in a2a calls
|
|
60
|
+
for node in ast.walk(tree):
|
|
61
|
+
if isinstance(node, ast.Call):
|
|
62
|
+
# Heuristic: passing a variable named 'state', 'context', or 'message_log'
|
|
63
|
+
for arg in node.args:
|
|
64
|
+
if isinstance(arg, ast.Name) and arg.id in ["state", "full_context", "messages", "history"]:
|
|
65
|
+
# Detection of "Chatter Bloat"
|
|
66
|
+
findings.append(AuditFinding(
|
|
67
|
+
category="📉 A2A Efficiency",
|
|
68
|
+
title="A2A Chatter Bloat Detected",
|
|
69
|
+
description=f"Passing entire variable '{arg.id}' to tool/agent call. This introduces high latency and token waste.",
|
|
70
|
+
impact="MEDIUM",
|
|
71
|
+
roi="Reduces token cost and latency by 30-50% through surgical state passing.",
|
|
72
|
+
line_number=node.lineno,
|
|
73
|
+
file_path=file_path
|
|
74
|
+
))
|
|
75
|
+
|
|
76
|
+
return findings
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import ast
|
|
2
|
+
import re
|
|
3
|
+
from typing import List
|
|
4
|
+
from .base import BaseAuditor, AuditFinding
|
|
5
|
+
|
|
6
|
+
class SovereigntyAuditor(BaseAuditor):
|
|
7
|
+
"""
|
|
8
|
+
v1.1 Phase 6: The Sovereign.
|
|
9
|
+
Audits for Multi-Cloud Portability and Data Residency (GDPR/EU Sovereign).
|
|
10
|
+
Detects Vendor Lock-in (Hardcoded Provider Specifics).
|
|
11
|
+
"""
|
|
12
|
+
VULNERABLE_PATTERNS = [
|
|
13
|
+
(r"project[-_]id\s*=\s*['\"][\w-]+['\"]", "Hardcoded GCP Project ID. Use environment variables for portability."),
|
|
14
|
+
(r"region\s*=\s*['\"]us-[\w-]+['\"]", "Hardcoded US Region. Risk to EU Data Sovereignty if not configured per environment."),
|
|
15
|
+
(r"aws_[\w-]+\s*=\s*['\"]", "AWS-specific credential detected. Ensure abstraction layer for Multi-Cloud."),
|
|
16
|
+
(r"azure_[\w-]+\s*=\s*['\"]", "Azure-specific endpoint detected. Risk of vendor lock-in.")
|
|
17
|
+
]
|
|
18
|
+
|
|
19
|
+
def audit(self, tree: ast.AST, content: str, file_path: str) -> List[AuditFinding]:
|
|
20
|
+
findings = []
|
|
21
|
+
|
|
22
|
+
# 1. Vendor Lock-in Check (Hardcoded IDs/Regions)
|
|
23
|
+
for pattern, reason in self.VULNERABLE_PATTERNS:
|
|
24
|
+
matches = re.finditer(pattern, content)
|
|
25
|
+
for match in matches:
|
|
26
|
+
# Find the line number based on char offset
|
|
27
|
+
line_no = content.count('\n', 0, match.start()) + 1
|
|
28
|
+
findings.append(AuditFinding(
|
|
29
|
+
category="🌍 Sovereignty",
|
|
30
|
+
title="Vendor Lock-in Risk",
|
|
31
|
+
description=reason,
|
|
32
|
+
impact="MEDIUM",
|
|
33
|
+
roi="Enables Multi-Cloud failover and EU sovereignty compliance.",
|
|
34
|
+
line_number=line_no,
|
|
35
|
+
file_path=file_path
|
|
36
|
+
))
|
|
37
|
+
|
|
38
|
+
# 2. Data Residency Audit (Sovereign Cloud patterns)
|
|
39
|
+
if "google.cloud" in content and "europe-" not in content.lower():
|
|
40
|
+
if "gdpr" in content.lower() or "compliance" in content.lower():
|
|
41
|
+
findings.append(AuditFinding(
|
|
42
|
+
category="🌍 Sovereignty",
|
|
43
|
+
title="EU Data Sovereignty Gap",
|
|
44
|
+
description="Compliance code detected but no European region routing found. Risk of non-compliance with EU data residency laws.",
|
|
45
|
+
impact="HIGH",
|
|
46
|
+
roi="Prevents multi-million Euro GDPR fines.",
|
|
47
|
+
file_path=file_path
|
|
48
|
+
))
|
|
49
|
+
|
|
50
|
+
# 3. Cloud Agnostic Interface Audit
|
|
51
|
+
direct_sdks = ["vertexai", "boto3", "azure-identity"]
|
|
52
|
+
for sdk in direct_sdks:
|
|
53
|
+
if f"import {sdk}" in content or f"from {sdk}" in content:
|
|
54
|
+
findings.append(AuditFinding(
|
|
55
|
+
category="🌍 Sovereignty",
|
|
56
|
+
title="Direct Vendor SDK Exposure",
|
|
57
|
+
description=f"Directly importing '{sdk}'. Consider wrapping in a provider-agnostic bridge to allow Multi-Cloud mobility.",
|
|
58
|
+
impact="LOW",
|
|
59
|
+
roi="Reduces refactoring cost during platform migration.",
|
|
60
|
+
file_path=file_path
|
|
61
|
+
))
|
|
62
|
+
|
|
63
|
+
# 4. v1.3 Strategic Exit: TCO & Migration Plan
|
|
64
|
+
if any(x in content.lower() for x in ["vertexai", "google.cloud"]):
|
|
65
|
+
findings.append(AuditFinding(
|
|
66
|
+
category="🌍 Sovereignty",
|
|
67
|
+
title="Strategic Exit Plan (Cloud)",
|
|
68
|
+
description="Detected hardcoded cloud dependencies. For a 'Category Killer' grade, implement an abstraction layer that allows switching to Gemma 2 on GKE.",
|
|
69
|
+
impact="INFO",
|
|
70
|
+
roi="Estimated 12% OpEx reduction via open-source pivot. Exit effort: ~14 lines of code.",
|
|
71
|
+
file_path=file_path
|
|
72
|
+
))
|
|
73
|
+
|
|
74
|
+
return findings
|