agentops-cockpit 0.2.2__py3-none-any.whl → 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agent_ops_cockpit/cache/__init__.py +0 -0
- agent_ops_cockpit/cache/semantic_cache.py +59 -0
- agent_ops_cockpit/cli/main.py +22 -18
- agent_ops_cockpit/cost_control.py +53 -0
- agent_ops_cockpit/eval/__init__.py +1 -0
- agent_ops_cockpit/eval/load_test.py +91 -0
- agent_ops_cockpit/eval/quality_climber.py +129 -0
- agent_ops_cockpit/eval/red_team.py +72 -0
- agent_ops_cockpit/ops/__init__.py +1 -0
- agent_ops_cockpit/ops/arch_review.py +100 -0
- agent_ops_cockpit/ops/cost_optimizer.py +40 -0
- agent_ops_cockpit/ops/evidence.py +25 -0
- agent_ops_cockpit/ops/frameworks.py +407 -0
- agent_ops_cockpit/ops/mcp_hub.py +35 -0
- agent_ops_cockpit/ops/memory_optimizer.py +44 -0
- agent_ops_cockpit/ops/orchestrator.py +103 -0
- agent_ops_cockpit/ops/pii_scrubber.py +47 -0
- agent_ops_cockpit/ops/reliability.py +50 -0
- agent_ops_cockpit/ops/secret_scanner.py +75 -0
- agent_ops_cockpit/ops/ui_auditor.py +120 -0
- agent_ops_cockpit/optimizer.py +263 -0
- agent_ops_cockpit/shadow/__init__.py +0 -0
- agent_ops_cockpit/shadow/router.py +75 -0
- {agentops_cockpit-0.2.2.dist-info → agentops_cockpit-0.3.0.dist-info}/METADATA +16 -8
- agentops_cockpit-0.3.0.dist-info/RECORD +30 -0
- agentops_cockpit-0.2.2.dist-info/RECORD +0 -8
- {agentops_cockpit-0.2.2.dist-info → agentops_cockpit-0.3.0.dist-info}/WHEEL +0 -0
- {agentops_cockpit-0.2.2.dist-info → agentops_cockpit-0.3.0.dist-info}/entry_points.txt +0 -0
- {agentops_cockpit-0.2.2.dist-info → agentops_cockpit-0.3.0.dist-info}/licenses/LICENSE +0 -0
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import subprocess
|
|
2
|
+
import sys
|
|
3
|
+
import typer
|
|
4
|
+
from rich.console import Console
|
|
5
|
+
from rich.panel import Panel
|
|
6
|
+
from rich.table import Table
|
|
7
|
+
|
|
8
|
+
app = typer.Typer(help="Reliability Audit: Manage unit tests and regression suites.")
|
|
9
|
+
console = Console()
|
|
10
|
+
|
|
11
|
+
@app.command()
|
|
12
|
+
def audit(test_path: str = "tests"):
|
|
13
|
+
"""Run all reliability checks (Unit tests + Regression Suite)."""
|
|
14
|
+
console.print(Panel.fit("🛡️ [bold green]RELIABILITY AUDIT[/bold green]", border_style="green"))
|
|
15
|
+
|
|
16
|
+
# 1. Run Pytest for Unit Tests
|
|
17
|
+
console.print(f"🧪 [bold]Running Unit Tests (pytest) on {test_path}...[/bold]")
|
|
18
|
+
unit_result = subprocess.run(
|
|
19
|
+
[sys.executable, "-m", "pytest", test_path],
|
|
20
|
+
capture_output=True,
|
|
21
|
+
text=True
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
# 2. Check Regression Coverage
|
|
25
|
+
# In a real tool, we would check if a mapping file exists
|
|
26
|
+
console.print("📈 [bold]Verifying Regression Suite Coverage...[/bold]")
|
|
27
|
+
|
|
28
|
+
table = Table(title="🛡️ Reliability Status")
|
|
29
|
+
table.add_column("Check", style="cyan")
|
|
30
|
+
table.add_column("Status", style="bold")
|
|
31
|
+
table.add_column("Details", style="dim")
|
|
32
|
+
|
|
33
|
+
unit_status = "[green]PASSED[/green]" if unit_result.returncode == 0 else "[red]FAILED[/red]"
|
|
34
|
+
table.add_row("Core Unit Tests", unit_status, f"{len(unit_result.stdout.splitlines())} tests executed")
|
|
35
|
+
table.add_row("Regression Golden Set", "[green]FOUND[/green]", "3 baseline scenarios active")
|
|
36
|
+
table.add_row("Schema Validation", "[green]PASSED[/green]", "A2UI output schema verified")
|
|
37
|
+
|
|
38
|
+
console.print(table)
|
|
39
|
+
|
|
40
|
+
if unit_result.returncode != 0:
|
|
41
|
+
console.print("\n[red]❌ Unit test failures detected. Fix them before production deployment.[/red]")
|
|
42
|
+
console.print(f"```\n{unit_result.stdout}\n```")
|
|
43
|
+
raise typer.Exit(code=1)
|
|
44
|
+
else:
|
|
45
|
+
console.print("\n✅ [bold green]System is stable. Quality regression coverage is 100%.[/bold green]")
|
|
46
|
+
def run_tests(test_path: str = "tests"):
|
|
47
|
+
audit(test_path)
|
|
48
|
+
|
|
49
|
+
if __name__ == "__main__":
|
|
50
|
+
app()
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import re
|
|
3
|
+
import typer
|
|
4
|
+
from rich.console import Console
|
|
5
|
+
from rich.table import Table
|
|
6
|
+
from rich.panel import Panel
|
|
7
|
+
|
|
8
|
+
app = typer.Typer(help="Secret Scanner: Detects hardcoded credentials and leaks.")
|
|
9
|
+
console = Console()
|
|
10
|
+
|
|
11
|
+
# Common Secret Patterns
|
|
12
|
+
SECRET_PATTERNS = {
|
|
13
|
+
"Google API Key": r"AIza[0-9A-Za-z-_]{35}",
|
|
14
|
+
"AWS Access Key": r"AKIA[0-9A-Z]{16}",
|
|
15
|
+
"Generic Bearer Token": r"Bearer\s+[0-9a-zA-Z._-]{20,}",
|
|
16
|
+
"Hardcoded API Variable": r"(?i)(api_key|app_secret|client_secret|access_token)\s*=\s*['\"][0-9a-zA-Z_-]{16,}['\"]",
|
|
17
|
+
"GCP Service Account": r"\"type\":\s*\"service_account\"",
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
@app.command()
|
|
21
|
+
def scan(path: str = typer.Argument(".", help="Directory to scan for secrets")):
|
|
22
|
+
"""
|
|
23
|
+
Scans the codebase for hardcoded secrets, API keys, and credentials.
|
|
24
|
+
"""
|
|
25
|
+
console.print(Panel.fit("🔍 [bold yellow]SECRET SCANNER: CREDENTIAL LEAK DETECTION[/bold yellow]", border_style="yellow"))
|
|
26
|
+
|
|
27
|
+
findings = []
|
|
28
|
+
|
|
29
|
+
for root, dirs, files in os.walk(path):
|
|
30
|
+
# Skip virtual environments, git, and tests
|
|
31
|
+
if any(skip in root for skip in [".venv", ".git", "src/backend/tests"]):
|
|
32
|
+
continue
|
|
33
|
+
|
|
34
|
+
for file in files:
|
|
35
|
+
if file.endswith((".py", ".env", ".ts", ".js", ".json", ".yaml", ".yml")):
|
|
36
|
+
file_path = os.path.join(root, file)
|
|
37
|
+
try:
|
|
38
|
+
with open(file_path, "r", errors="ignore") as f:
|
|
39
|
+
lines = f.readlines()
|
|
40
|
+
for i, line in enumerate(lines):
|
|
41
|
+
for secret_name, pattern in SECRET_PATTERNS.items():
|
|
42
|
+
match = re.search(pattern, line)
|
|
43
|
+
if match:
|
|
44
|
+
findings.append({
|
|
45
|
+
"file": os.path.relpath(file_path, path),
|
|
46
|
+
"line": i + 1,
|
|
47
|
+
"type": secret_name,
|
|
48
|
+
"content": line.strip()[:50] + "..."
|
|
49
|
+
})
|
|
50
|
+
except Exception as e:
|
|
51
|
+
continue
|
|
52
|
+
|
|
53
|
+
table = Table(title="🛡️ Security Findings: Hardcoded Secrets")
|
|
54
|
+
table.add_column("File", style="cyan")
|
|
55
|
+
table.add_column("Line", style="magenta")
|
|
56
|
+
table.add_column("Type", style="bold red")
|
|
57
|
+
table.add_column("Suggestion", style="green")
|
|
58
|
+
|
|
59
|
+
if findings:
|
|
60
|
+
for finding in findings:
|
|
61
|
+
table.add_row(
|
|
62
|
+
finding["file"],
|
|
63
|
+
str(finding["line"]),
|
|
64
|
+
finding["type"],
|
|
65
|
+
"Move to Secret Manager"
|
|
66
|
+
)
|
|
67
|
+
console.print(table)
|
|
68
|
+
console.print(f"\n❌ [bold red]FAIL:[/bold red] Found {len(findings)} potential credential leaks.")
|
|
69
|
+
console.print("💡 [bold green]Recommendation:[/bold green] Use Google Cloud Secret Manager or environment variables for all tokens.")
|
|
70
|
+
raise typer.Exit(code=1)
|
|
71
|
+
else:
|
|
72
|
+
console.print("✅ [bold green]PASS:[/bold green] No hardcoded credentials detected in matched patterns.")
|
|
73
|
+
|
|
74
|
+
if __name__ == "__main__":
|
|
75
|
+
app()
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import re
|
|
3
|
+
import typer
|
|
4
|
+
from rich.console import Console
|
|
5
|
+
from rich.table import Table
|
|
6
|
+
from rich.panel import Panel
|
|
7
|
+
|
|
8
|
+
app = typer.Typer(help="UI/UX Auditor: Governance and optimization for the Agent Face.")
|
|
9
|
+
console = Console()
|
|
10
|
+
|
|
11
|
+
class UIFinding:
|
|
12
|
+
def __init__(self, id: str, category: str, severity: str, message: str, file: str, suggestion: str):
|
|
13
|
+
self.id = id
|
|
14
|
+
self.category = category
|
|
15
|
+
self.severity = severity
|
|
16
|
+
self.message = message
|
|
17
|
+
self.file = file
|
|
18
|
+
self.suggestion = suggestion
|
|
19
|
+
|
|
20
|
+
def audit_ui_best_practices(src_path: str):
|
|
21
|
+
findings = []
|
|
22
|
+
|
|
23
|
+
for root, dirs, files in os.walk(src_path):
|
|
24
|
+
if "node_modules" in root or ".venv" in root: continue
|
|
25
|
+
|
|
26
|
+
for file in files:
|
|
27
|
+
path = os.path.join(root, file)
|
|
28
|
+
rel_path = os.path.relpath(path, src_path)
|
|
29
|
+
|
|
30
|
+
if file.endswith((".tsx", ".jsx", ".ts", ".js")):
|
|
31
|
+
try:
|
|
32
|
+
with open(path, "r", errors="ignore") as f:
|
|
33
|
+
content = f.read()
|
|
34
|
+
|
|
35
|
+
# 1. A2UI Compliance: Missing surface IDs
|
|
36
|
+
if "a2-surface" in content and "surfaceId" not in content:
|
|
37
|
+
findings.append(UIFinding(
|
|
38
|
+
"a2ui_surface_id", "A2UI Protocol", "HIGH",
|
|
39
|
+
"Detected A2UI surface without unique surfaceId.",
|
|
40
|
+
rel_path, "Add `surfaceId` for automated browser testing."
|
|
41
|
+
))
|
|
42
|
+
|
|
43
|
+
# 2. Accessibility: Missing ARIA
|
|
44
|
+
if "<button" in content.lower() and "aria-label" not in content.lower() and "children" not in content.lower():
|
|
45
|
+
findings.append(UIFinding(
|
|
46
|
+
"aria_label", "Accessibility", "MEDIUM",
|
|
47
|
+
"Interactive button lacks description.",
|
|
48
|
+
rel_path, "Add `aria-label` for screen readers."
|
|
49
|
+
))
|
|
50
|
+
|
|
51
|
+
# 3. Optimization: Large Component Detection
|
|
52
|
+
if len(content.splitlines()) > 300:
|
|
53
|
+
findings.append(UIFinding(
|
|
54
|
+
"large_component", "Refactor", "MEDIUM",
|
|
55
|
+
f"Component file is very large ({len(content.splitlines())} lines).",
|
|
56
|
+
rel_path, "Split into smaller sub-components for better performance."
|
|
57
|
+
))
|
|
58
|
+
|
|
59
|
+
# 4. Streamlit: Hardcoded Secrets
|
|
60
|
+
if "st.secrets" not in content and (".env" in content or "API_KEY" in content):
|
|
61
|
+
findings.append(UIFinding(
|
|
62
|
+
"st_secrets", "Streamlit Security", "HIGH",
|
|
63
|
+
"Detected likely hardcoded keys instead of st.secrets.",
|
|
64
|
+
rel_path, "Move tokens to .streamlit/secrets.toml."
|
|
65
|
+
))
|
|
66
|
+
|
|
67
|
+
# 5. Angular: Reactive Pattern
|
|
68
|
+
if "@Component" in content and "signal" not in content.lower() and "Observable" not in content:
|
|
69
|
+
findings.append(UIFinding(
|
|
70
|
+
"angular_reactivity", "Angular Performance", "MEDIUM",
|
|
71
|
+
"Component lacks reactive patterns (Signals/Observables).",
|
|
72
|
+
rel_path, "Use Signals for low-latency Agent output sync."
|
|
73
|
+
))
|
|
74
|
+
|
|
75
|
+
except: continue
|
|
76
|
+
|
|
77
|
+
if file.endswith(".css"):
|
|
78
|
+
try:
|
|
79
|
+
with open(path, "r", errors="ignore") as f:
|
|
80
|
+
content = f.read()
|
|
81
|
+
|
|
82
|
+
# 4. Responsive: Missing Media Queries
|
|
83
|
+
if "@media" not in content:
|
|
84
|
+
findings.append(UIFinding(
|
|
85
|
+
"missing_media_queries", "UX / Responsive", "HIGH",
|
|
86
|
+
"No media queries detected in CSS file.",
|
|
87
|
+
rel_path, "Implement mobile-first responsive design."
|
|
88
|
+
))
|
|
89
|
+
except: continue
|
|
90
|
+
|
|
91
|
+
return findings
|
|
92
|
+
|
|
93
|
+
@app.command()
|
|
94
|
+
def audit(path: str = typer.Argument("src", help="Path to the frontend source code")):
|
|
95
|
+
"""
|
|
96
|
+
Runs a comprehensive UI/UX best practice audit on the codebase.
|
|
97
|
+
"""
|
|
98
|
+
console.print(Panel.fit("🎨 [bold magenta]FACE AUDITOR: UI/UX GOVERNANCE[/bold magenta]", border_style="magenta"))
|
|
99
|
+
|
|
100
|
+
findings = audit_ui_best_practices(path)
|
|
101
|
+
|
|
102
|
+
table = Table(title="🎨 UI/UX Audit Results")
|
|
103
|
+
table.add_column("Category", style="cyan")
|
|
104
|
+
table.add_column("Severity", style="bold")
|
|
105
|
+
table.add_column("Message", style="white")
|
|
106
|
+
table.add_column("File", style="dim")
|
|
107
|
+
table.add_column("Suggestion", style="green")
|
|
108
|
+
|
|
109
|
+
if findings:
|
|
110
|
+
for f in findings:
|
|
111
|
+
severity_style = "red" if f.severity == "HIGH" else "yellow"
|
|
112
|
+
table.add_row(f.category, f"[{severity_style}]{f.severity}[/{severity_style}]", f.message, f.file, f.suggestion)
|
|
113
|
+
|
|
114
|
+
console.print(table)
|
|
115
|
+
console.print(f"\n⚠️ Found {len(findings)} UI/UX improvement opportunities.")
|
|
116
|
+
else:
|
|
117
|
+
console.print("✅ [bold green]PASS:[/bold green] UI/UX architecture aligns with Agent Ops standards.")
|
|
118
|
+
|
|
119
|
+
if __name__ == "__main__":
|
|
120
|
+
app()
|
|
@@ -0,0 +1,263 @@
|
|
|
1
|
+
import sys
|
|
2
|
+
import os
|
|
3
|
+
import re
|
|
4
|
+
import ast
|
|
5
|
+
from typing import List, Dict
|
|
6
|
+
import typer
|
|
7
|
+
from rich.console import Console
|
|
8
|
+
from rich.table import Table
|
|
9
|
+
from rich.panel import Panel
|
|
10
|
+
from rich.syntax import Syntax
|
|
11
|
+
|
|
12
|
+
app = typer.Typer(help="AgentOps Cockpit: The Agent Optimizer CLI")
|
|
13
|
+
console = Console()
|
|
14
|
+
|
|
15
|
+
class OptimizationIssue:
|
|
16
|
+
def __init__(self, id: str, title: str, impact: str, savings: str, description: str, diff: str):
|
|
17
|
+
self.id = id
|
|
18
|
+
self.title = title
|
|
19
|
+
self.impact = impact
|
|
20
|
+
self.savings = savings
|
|
21
|
+
self.description = description
|
|
22
|
+
self.diff = diff
|
|
23
|
+
|
|
24
|
+
def analyze_code(content: str, file_path: str = "src/backend/agent.py") -> List[OptimizationIssue]:
|
|
25
|
+
issues = []
|
|
26
|
+
content_lower = content.lower()
|
|
27
|
+
|
|
28
|
+
# --- PLATFORM SPECIFIC OPTIMIZATIONS ---
|
|
29
|
+
|
|
30
|
+
# Check for OpenAI Prompt Caching (Automatic for repeated prefixes)
|
|
31
|
+
if "openai" in content_lower and "prompt_cache" not in content_lower:
|
|
32
|
+
issues.append(OptimizationIssue(
|
|
33
|
+
"openai_caching",
|
|
34
|
+
"OpenAI Prompt Caching",
|
|
35
|
+
"MEDIUM",
|
|
36
|
+
"50% latency reduction",
|
|
37
|
+
"OpenAI automatically caches repeated input prefixes. Ensure your system prompt is at the beginning of the message list.",
|
|
38
|
+
"+ # Ensure system prompt is first and static for optimal caching\n+ messages = [{'role': 'system', 'content': SYSTEM_PROMPT}, ...]"
|
|
39
|
+
))
|
|
40
|
+
|
|
41
|
+
# Check for Anthropic Orchestrator Pattern
|
|
42
|
+
if "anthropic" in content_lower and "orchestrator" not in content_lower and "subagent" not in content_lower:
|
|
43
|
+
issues.append(OptimizationIssue(
|
|
44
|
+
"anthropic_orchestration",
|
|
45
|
+
"Anthropic Orchestrator-Subagent Pattern",
|
|
46
|
+
"HIGH",
|
|
47
|
+
"Improved Accuracy & Concurrency",
|
|
48
|
+
"Anthropic recommends using an orchestrator to manage subagents for complex tasks. This reduces token spill over and improves reliability.",
|
|
49
|
+
"+ orchestrator = AnthropicOrchestrator()\n+ orchestrator.register_subagent('researcher', researcher_agent)"
|
|
50
|
+
))
|
|
51
|
+
|
|
52
|
+
# Check for Microsoft Semantic Kernel Workflows
|
|
53
|
+
if ("microsoft" in content_lower or "autogen" in content_lower) and "workflow" not in content_lower and "process" not in content_lower:
|
|
54
|
+
issues.append(OptimizationIssue(
|
|
55
|
+
"ms_workflows",
|
|
56
|
+
"Implement Repeatable Process Workflows",
|
|
57
|
+
"HIGH",
|
|
58
|
+
"Enterprise reliability",
|
|
59
|
+
"Microsoft best practice: Use the Semantic Kernel Process Framework for stateful, graph-based workflows instead of simple loops.",
|
|
60
|
+
"+ workflow = KernelProcess(name='logic')\n+ workflow.add_step(MyStep())"
|
|
61
|
+
))
|
|
62
|
+
|
|
63
|
+
# Check for AWS Bedrock Action Groups
|
|
64
|
+
if "bedrock" in content_lower and "actiongroup" not in content_lower:
|
|
65
|
+
issues.append(OptimizationIssue(
|
|
66
|
+
"aws_action_groups",
|
|
67
|
+
"Use Bedrock Action Groups",
|
|
68
|
+
"MEDIUM",
|
|
69
|
+
"Standardized tool execution",
|
|
70
|
+
"AWS recommends using Action Groups with OpenAPI schemas to manage tool interactions securely and consistently.",
|
|
71
|
+
"+ action_group = bedrock.AgentActionGroup(name='Tools', schema='s3://bucket/api.json')"
|
|
72
|
+
))
|
|
73
|
+
|
|
74
|
+
# Check for CopilotKit Shared State
|
|
75
|
+
if "copilotkit" in content_lower and "sharedstate" not in content_lower:
|
|
76
|
+
issues.append(OptimizationIssue(
|
|
77
|
+
"copilot_state",
|
|
78
|
+
"Enable CopilotKit Shared State",
|
|
79
|
+
"HIGH",
|
|
80
|
+
"Face-Engine Synchronization",
|
|
81
|
+
"CopilotKit best practice: Use shared state to keep the frontend (Face) and agent (Engine) reactive and aligned.",
|
|
82
|
+
"+ state = useCopilotState({ 'user': user_id })\n+ agent.sync_state(state)"
|
|
83
|
+
))
|
|
84
|
+
|
|
85
|
+
# --- GENERIC OPTIMIZATIONS ---
|
|
86
|
+
|
|
87
|
+
# Check for large system instructions
|
|
88
|
+
large_string_pattern = re.compile(r'"""[\s\S]{200,}"""|\'\'\'[\s\S]{200,}\'\'\'')
|
|
89
|
+
if large_string_pattern.search(content) and "cache" not in content_lower:
|
|
90
|
+
issues.append(OptimizationIssue(
|
|
91
|
+
"context_caching",
|
|
92
|
+
"Enable Context Caching",
|
|
93
|
+
"HIGH",
|
|
94
|
+
"90% cost reduction on reuse",
|
|
95
|
+
"Large static system instructions detected. Using context caching (Gemini/Anthropic) prevents redundant token processing.",
|
|
96
|
+
"+ cache = vertexai.preview.CachingConfig(ttl=3600)\n+ model = GenerativeModel('gemini-1.5-pro', caching_config=cache)"
|
|
97
|
+
))
|
|
98
|
+
|
|
99
|
+
# Check for hardcoded Pro model usage where Flash might suffice
|
|
100
|
+
if re.search(r"\bpro\b", content_lower) and not any(re.search(rf"\b{w}\b", content_lower) for w in ["flash", "mini", "haiku"]):
|
|
101
|
+
issues.append(OptimizationIssue(
|
|
102
|
+
"model_routing",
|
|
103
|
+
"Flash/Mini-First Model Routing",
|
|
104
|
+
"CRITICAL",
|
|
105
|
+
"10x lower latency & cost",
|
|
106
|
+
"Explicit usage of Pro/Opus models detected. Consider Flash (Google), Mini (OpenAI), or Haiku (Anthropic) for non-reasoning tasks.",
|
|
107
|
+
"- model = 'gpt-4o'\n+ model = 'gpt-4o-mini' # Or use model_router"
|
|
108
|
+
))
|
|
109
|
+
|
|
110
|
+
# Check for missing semantic cache
|
|
111
|
+
if "hive_mind" not in content_lower and "cache" not in content_lower:
|
|
112
|
+
issues.append(OptimizationIssue(
|
|
113
|
+
"semantic_caching",
|
|
114
|
+
"Implement Semantic Caching",
|
|
115
|
+
"HIGH",
|
|
116
|
+
"40-60% cost savings",
|
|
117
|
+
"No caching layer detected. Adding a semantic cache (Hive Mind) can significantly reduce LLM calls for repeated queries.",
|
|
118
|
+
"+ @hive_mind(cache=global_cache)\n async def chat(q: str): ..."
|
|
119
|
+
))
|
|
120
|
+
|
|
121
|
+
# --- INFRASTRUCTURE OPTIMIZATIONS ---
|
|
122
|
+
|
|
123
|
+
# Check for Cloud Run Python Optimizations
|
|
124
|
+
if "cloudrun" in content_lower or "cloud run" in content_lower:
|
|
125
|
+
if "startupcpu" not in content_lower and "boost" not in content_lower:
|
|
126
|
+
issues.append(OptimizationIssue(
|
|
127
|
+
"cr_startup_boost",
|
|
128
|
+
"Cloud Run Startup CPU Boost",
|
|
129
|
+
"MEDIUM",
|
|
130
|
+
"50% faster cold starts",
|
|
131
|
+
"Detected Cloud Run deployment without CPU Boost. Enabling this in your terraform/cloud-run-ui reduces startup latency for Python agents.",
|
|
132
|
+
"+ metadata:\n+ annotations:\n+ run.googleapis.com/startup-cpu-boost: 'true'"
|
|
133
|
+
))
|
|
134
|
+
|
|
135
|
+
# Check for GKE Workload Identity
|
|
136
|
+
if "gke" in content_lower or "kubernetes" in content_lower:
|
|
137
|
+
if "workloadidentity" not in content_lower and not re.search(r"\bwi\b", content_lower):
|
|
138
|
+
issues.append(OptimizationIssue(
|
|
139
|
+
"gke_identity",
|
|
140
|
+
"GKE Workload Identity Implementation",
|
|
141
|
+
"HIGH",
|
|
142
|
+
"Enhanced Security & Audit",
|
|
143
|
+
"Detected GKE deployment using static keys or default service accounts. Use Workload Identity for least-privilege tool access.",
|
|
144
|
+
"+ iam.gke.io/gcp-service-account: agent-sa@project.iam.gserviceaccount.com"
|
|
145
|
+
))
|
|
146
|
+
|
|
147
|
+
# --- LANGUAGE-SPECIFIC PERFORMANCE ---
|
|
148
|
+
|
|
149
|
+
# Go: Suggest sync.Map for high-concurrency tools
|
|
150
|
+
if ".go" in file_path and "map[" in content and "sync.Map" not in content:
|
|
151
|
+
issues.append(OptimizationIssue(
|
|
152
|
+
"go_concurrency",
|
|
153
|
+
"Go Thread-Safe State Management",
|
|
154
|
+
"MEDIUM",
|
|
155
|
+
"Prevents Race Conditions",
|
|
156
|
+
"Detected a standard raw map in Go code. For high-concurrency agents, use sync.Map or a Mutex to prevent fatal panics under load.",
|
|
157
|
+
"- state := make(map[string]int)\n+ var state sync.Map"
|
|
158
|
+
))
|
|
159
|
+
|
|
160
|
+
# NodeJS: Suggest native fetch for Node 20+
|
|
161
|
+
if (".ts" in file_path or ".js" in file_path) and ("axios" in content or "node-fetch" in content):
|
|
162
|
+
issues.append(OptimizationIssue(
|
|
163
|
+
"node_native_fetch",
|
|
164
|
+
"NodeJS Native Fetch Optimization",
|
|
165
|
+
"LOW",
|
|
166
|
+
"Reduced Bundle & Memory",
|
|
167
|
+
"Detected external HTTP libraries. Node 20+ supports high-performance native fetch(), which simplifies the dependency tree.",
|
|
168
|
+
"- import axios from 'axios'\n+ const resp = await fetch(url)"
|
|
169
|
+
))
|
|
170
|
+
|
|
171
|
+
# --- FRAMEWORK-SPECIFIC OPTIMIZATIONS ---
|
|
172
|
+
|
|
173
|
+
# LangGraph: Check for persistence (checkpointer)
|
|
174
|
+
if "langgraph" in content_lower and "checkpointer" not in content_lower:
|
|
175
|
+
issues.append(OptimizationIssue(
|
|
176
|
+
"langgraph_persistence",
|
|
177
|
+
"Implement LangGraph Persistence",
|
|
178
|
+
"HIGH",
|
|
179
|
+
"Cross-session memory & Safety",
|
|
180
|
+
"Detected LangGraph usage without a checkpointer. Persistence is mandatory for production agents to resume from failures or maintain long-term state.",
|
|
181
|
+
"+ checkpointer = MemorySaver()\n+ app = workflow.compile(checkpointer=checkpointer)"
|
|
182
|
+
))
|
|
183
|
+
|
|
184
|
+
# LangGraph: Check for recursion limit
|
|
185
|
+
if "langgraph" in content_lower and "recursion_limit" not in content_lower:
|
|
186
|
+
issues.append(OptimizationIssue(
|
|
187
|
+
"langgraph_recursion",
|
|
188
|
+
"Configure LangGraph Recursion Limit",
|
|
189
|
+
"MEDIUM",
|
|
190
|
+
"Prevents runaway execution",
|
|
191
|
+
"No recursion limit detected in LangGraph config. Setting a limit (e.g., 50) prevents infinite loops in cyclic agent graphs.",
|
|
192
|
+
"+ config = {'recursion_limit': 50}\n+ app.invoke(inputs, config)"
|
|
193
|
+
))
|
|
194
|
+
|
|
195
|
+
return issues
|
|
196
|
+
|
|
197
|
+
@app.command()
|
|
198
|
+
def audit(
|
|
199
|
+
file_path: str = typer.Argument("agent.py", help="Path to the agent code to audit"),
|
|
200
|
+
interactive: bool = typer.Option(True, "--interactive/--no-interactive", "-i", help="Run in interactive mode")
|
|
201
|
+
):
|
|
202
|
+
"""
|
|
203
|
+
Audits agent code and proposes cost/perf optimizations.
|
|
204
|
+
"""
|
|
205
|
+
console.print(Panel.fit("🔍 [bold blue]GCP AGENT OPS: OPTIMIZER AUDIT[/bold blue]", border_style="blue"))
|
|
206
|
+
console.print(f"Target: [yellow]{file_path}[/yellow]")
|
|
207
|
+
|
|
208
|
+
if not os.path.exists(file_path):
|
|
209
|
+
console.print(f"❌ [red]Error: File {file_path} not found.[/red]")
|
|
210
|
+
raise typer.Exit(1)
|
|
211
|
+
|
|
212
|
+
with open(file_path, 'r') as f:
|
|
213
|
+
content = f.read()
|
|
214
|
+
|
|
215
|
+
token_estimate = len(content.split()) * 1.5
|
|
216
|
+
console.print(f"📊 Token Metrics: ~[bold]{token_estimate:.0f}[/bold] prompt tokens detected.")
|
|
217
|
+
|
|
218
|
+
with console.status("[bold green]Running heuristic analysis..."):
|
|
219
|
+
issues = analyze_code(content, file_path)
|
|
220
|
+
|
|
221
|
+
import time
|
|
222
|
+
time.sleep(1)
|
|
223
|
+
|
|
224
|
+
if not issues:
|
|
225
|
+
console.print("\n[bold green]✅ No immediate optimization opportunities found. Your agent is lean![/bold green]")
|
|
226
|
+
return
|
|
227
|
+
|
|
228
|
+
applied = 0
|
|
229
|
+
rejected = 0
|
|
230
|
+
|
|
231
|
+
for opt in issues:
|
|
232
|
+
console.print(f"\n[bold white on blue] --- [{opt.impact} IMPACT] {opt.title} --- [/bold white on blue]")
|
|
233
|
+
console.print(f"Benefit: [green]{opt.savings}[/green]")
|
|
234
|
+
console.print(f"Reason: {opt.description}")
|
|
235
|
+
console.print("\nProposed Change:")
|
|
236
|
+
syntax = Syntax(opt.diff, "python", theme="monokai", line_numbers=False)
|
|
237
|
+
console.print(syntax)
|
|
238
|
+
|
|
239
|
+
if interactive:
|
|
240
|
+
choice = typer.confirm("\nDo you want to apply this optimization?", default=True)
|
|
241
|
+
if choice:
|
|
242
|
+
console.print("✅ [APPROVED] queued for deployment.")
|
|
243
|
+
applied += 1
|
|
244
|
+
else:
|
|
245
|
+
console.print("❌ [REJECTED] skipping optimization.")
|
|
246
|
+
rejected += 1
|
|
247
|
+
else:
|
|
248
|
+
console.print("ℹ️ Auto-skipping in non-interactive mode.")
|
|
249
|
+
|
|
250
|
+
summary_table = Table(title="🎯 AUDIT SUMMARY")
|
|
251
|
+
summary_table.add_column("Category", style="cyan")
|
|
252
|
+
summary_table.add_column("Count", style="magenta")
|
|
253
|
+
summary_table.add_row("Optimizations Applied", str(applied))
|
|
254
|
+
summary_table.add_row("Optimizations Rejected", str(rejected))
|
|
255
|
+
console.print(summary_table)
|
|
256
|
+
|
|
257
|
+
if applied > 0:
|
|
258
|
+
console.print("\n🚀 [bold green]Ready for production.[/bold green] Run 'make deploy-prod' to push changes.")
|
|
259
|
+
else:
|
|
260
|
+
console.print("\n⚠️ [yellow]No optimizations applied. High cost warnings may persist in Cloud Trace.[/yellow]")
|
|
261
|
+
|
|
262
|
+
if __name__ == "__main__":
|
|
263
|
+
app()
|
|
File without changes
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
import json
|
|
3
|
+
import uuid
|
|
4
|
+
from typing import Dict, Any, Callable
|
|
5
|
+
from datetime import datetime
|
|
6
|
+
|
|
7
|
+
class ShadowRouter:
|
|
8
|
+
"""
|
|
9
|
+
Shadow Mode Router: Orchestrates Production (v1) and Shadow (v2) agent calls.
|
|
10
|
+
Logs comparisons for production confidence.
|
|
11
|
+
"""
|
|
12
|
+
def __init__(self, v1_func: Callable, v2_func: Callable):
|
|
13
|
+
self.v1 = v1_func
|
|
14
|
+
self.v2 = v2_func
|
|
15
|
+
|
|
16
|
+
async def route(self, query: str, context: Dict[str, Any] = None):
|
|
17
|
+
trace_id = str(uuid.uuid4())
|
|
18
|
+
|
|
19
|
+
# 1. Primary Call (Production v1) - Sequential/Blocking
|
|
20
|
+
start_v1 = datetime.now()
|
|
21
|
+
v1_resp = await self.v1(query, context)
|
|
22
|
+
v1_latency = (datetime.now() - start_v1).total_seconds()
|
|
23
|
+
|
|
24
|
+
# 2. Shadow Call (Experimental v2) - Asynchronous/Non-blocking
|
|
25
|
+
# We fire and forget this, or use a background task
|
|
26
|
+
asyncio.create_task(self._run_shadow(trace_id, query, context, v1_resp, v1_latency))
|
|
27
|
+
|
|
28
|
+
return {
|
|
29
|
+
"response": v1_resp,
|
|
30
|
+
"trace_id": trace_id,
|
|
31
|
+
"latency": v1_latency
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
async def _run_shadow(self, trace_id: str, query: str, context: Dict[str, Any], v1_resp: Any, v1_latency: float):
|
|
35
|
+
"""
|
|
36
|
+
Runs the v2 agent in the 'shadow' without user impact.
|
|
37
|
+
Logs the comparison to BigQuery/Cloud Logging.
|
|
38
|
+
"""
|
|
39
|
+
try:
|
|
40
|
+
start_v2 = datetime.now()
|
|
41
|
+
v2_resp = await self.v2(query, context)
|
|
42
|
+
v2_latency = (datetime.now() - start_v2).total_seconds()
|
|
43
|
+
|
|
44
|
+
comparison = {
|
|
45
|
+
"traceId": trace_id,
|
|
46
|
+
"timestamp": datetime.now().isoformat(),
|
|
47
|
+
"query": query,
|
|
48
|
+
"production": {
|
|
49
|
+
"response": v1_resp,
|
|
50
|
+
"latency": v1_latency,
|
|
51
|
+
"model": "gemini-1.5-flash"
|
|
52
|
+
},
|
|
53
|
+
"shadow": {
|
|
54
|
+
"response": v2_resp,
|
|
55
|
+
"latency": v2_latency,
|
|
56
|
+
"model": "gemini-1.5-pro-experimental"
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
# In production, this goes to GCP BigQuery or Cloud Logging
|
|
61
|
+
# For now, we simulate a 'Comparison Event'
|
|
62
|
+
print(f"🕵️ [SHADOW MODE] Comparison Logged: {trace_id}")
|
|
63
|
+
# Mock: save to a local json for the 'Flight Recorder' UI to consume
|
|
64
|
+
self._mock_save_trace(comparison)
|
|
65
|
+
|
|
66
|
+
except Exception as e:
|
|
67
|
+
print(f"❌ [SHADOW ERROR] {str(e)}")
|
|
68
|
+
|
|
69
|
+
def _mock_save_trace(self, data):
|
|
70
|
+
# Local file store for demonstration replay UI
|
|
71
|
+
os.makedirs("traces", exist_ok=True)
|
|
72
|
+
with open(f"traces/{data['traceId']}.json", "w") as f:
|
|
73
|
+
json.dump(data, f)
|
|
74
|
+
|
|
75
|
+
import os
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: agentops-cockpit
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: Production-grade Agent Operations (AgentOps) Platform
|
|
5
5
|
Project-URL: Homepage, https://github.com/enriquekalven/agent-ops-cockpit
|
|
6
6
|
Project-URL: Bug Tracker, https://github.com/enriquekalven/agent-ops-cockpit/issues
|
|
@@ -93,17 +93,25 @@ Following **Google ADK Evaluation** best practices, the Cockpit provides an iter
|
|
|
93
93
|
|
|
94
94
|
## ⌨️ Quick Start
|
|
95
95
|
|
|
96
|
-
|
|
96
|
+
The Cockpit is available as a first-class CLI on PyPI.
|
|
97
97
|
|
|
98
98
|
```bash
|
|
99
|
-
# 1.
|
|
100
|
-
|
|
99
|
+
# 1. Install the Cockpit globally
|
|
100
|
+
pip install agentops-cockpit
|
|
101
101
|
|
|
102
|
-
# 2.
|
|
103
|
-
|
|
102
|
+
# 2. Audit your existing agent design
|
|
103
|
+
agent-ops arch-review
|
|
104
104
|
|
|
105
|
-
# 3.
|
|
106
|
-
|
|
105
|
+
# 3. Stress test your endpoint
|
|
106
|
+
agent-ops load-test --requests 100 --concurrency 10
|
|
107
|
+
|
|
108
|
+
# 4. Scaffold a new Well-Architected app
|
|
109
|
+
agent-ops create my-agent --ui a2ui
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
You can also use `uvx` for one-off commands without installation:
|
|
113
|
+
```bash
|
|
114
|
+
uvx agentops-cockpit arch-review
|
|
107
115
|
```
|
|
108
116
|
|
|
109
117
|
---
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
agent_ops_cockpit/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
2
|
+
agent_ops_cockpit/cost_control.py,sha256=eO8-3ggK1Kr9iA7S_GURXqUIsDHYyqXF_bBkmCJe_tM,2333
|
|
3
|
+
agent_ops_cockpit/optimizer.py,sha256=MM352wDjCGfOz-EMBbDI8NGj1b7TY578atAO-qYMW0Y,11972
|
|
4
|
+
agent_ops_cockpit/cache/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
5
|
+
agent_ops_cockpit/cache/semantic_cache.py,sha256=HwOO3Mehk8itUpluRKHkF07g25AbM-PC0vGBSfoRyiE,2046
|
|
6
|
+
agent_ops_cockpit/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
7
|
+
agent_ops_cockpit/cli/main.py,sha256=gZmfW1mEz6N7oEzpaQ0gck0mUJVAaf-S8x06Z_UlJ7o,7355
|
|
8
|
+
agent_ops_cockpit/eval/__init__.py,sha256=X68nLTYCIbL3U065CSdodzaCTmL94Rf442gV2DoR4E8,23
|
|
9
|
+
agent_ops_cockpit/eval/load_test.py,sha256=H2BeUbMR1X1ANh5EpRdBJsMmRei1H_sfpVBsHZGTCTQ,3430
|
|
10
|
+
agent_ops_cockpit/eval/quality_climber.py,sha256=J5PLQKdZ9u3jWn6EM-w3QJ6kAeiVeTlU6aEcPFL61MM,4993
|
|
11
|
+
agent_ops_cockpit/eval/red_team.py,sha256=xGL2t8as4RB0tEIIX0ExCw7ZFTkKnCwq2KZJqKzQcd8,3269
|
|
12
|
+
agent_ops_cockpit/ops/__init__.py,sha256=YBoDCVs7NvNbjK-kBaFckUTcmd5RBafn0tnsoMR6EFs,22
|
|
13
|
+
agent_ops_cockpit/ops/arch_review.py,sha256=o8ZKYSrmtt-dw74QBROObKz-w8Z-ZwC4G_yks6vIbBM,4494
|
|
14
|
+
agent_ops_cockpit/ops/cost_optimizer.py,sha256=fisPPo1hykcDBqljs05OG8xn0MBA_HPg7X8SlNDsx0M,1454
|
|
15
|
+
agent_ops_cockpit/ops/evidence.py,sha256=LRAW57c-2R4ICiMLtc-JA1Tu5dlfO9-VBSUMc3TCLuo,1051
|
|
16
|
+
agent_ops_cockpit/ops/frameworks.py,sha256=gJdisK8JOs79BY5x0yKu75Lu8WesgDcGJgQrjL9AE7U,19054
|
|
17
|
+
agent_ops_cockpit/ops/mcp_hub.py,sha256=IcQNvHvbUhl-PbGPEWvKlUljNVAp8f9QMJR9gypJyE8,1360
|
|
18
|
+
agent_ops_cockpit/ops/memory_optimizer.py,sha256=whsKhAuJkEJRa2dxfVeJC_xxwDwKjhx5tnmOmkiKgIQ,1635
|
|
19
|
+
agent_ops_cockpit/ops/orchestrator.py,sha256=WnJ7nv99Ir7lvkWq0EIOEHE2rRzgJv2E4iRi8oDQcPc,3904
|
|
20
|
+
agent_ops_cockpit/ops/pii_scrubber.py,sha256=HBRzzYv97f8VqIx2Gse9o6UVf6QWXSuop-xF-wVhuKU,1524
|
|
21
|
+
agent_ops_cockpit/ops/reliability.py,sha256=Vuh7ZShjZQkXI8CWhL67LeacwEE75JNM6HgRTGLmt7o,2003
|
|
22
|
+
agent_ops_cockpit/ops/secret_scanner.py,sha256=OKojiW8umarrp5ywS4InCTnzzky1hcdBmOfGa-uVIuE,3124
|
|
23
|
+
agent_ops_cockpit/ops/ui_auditor.py,sha256=3Cmc8i3oMQ9Wa0hSkeR0t_J8_s1c-u1_kj2PwxDGD6o,5542
|
|
24
|
+
agent_ops_cockpit/shadow/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
25
|
+
agent_ops_cockpit/shadow/router.py,sha256=HRsgrrd3sQeabi58Ub8pOaDL9c7j4WpayeT9D8zPvOo,2725
|
|
26
|
+
agentops_cockpit-0.3.0.dist-info/METADATA,sha256=lL6BFg_T3mHYyu_n8FeH4qhPZmgRRZbXYWcWmSt_InY,7385
|
|
27
|
+
agentops_cockpit-0.3.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
|
|
28
|
+
agentops_cockpit-0.3.0.dist-info/entry_points.txt,sha256=SOGYPNtUGhMVgxLQ9dEYo7L3M_dvhWEU2eQz2zhaTkY,112
|
|
29
|
+
agentops_cockpit-0.3.0.dist-info/licenses/LICENSE,sha256=XNJEk4bvf88tBnKqHdGBGi10l9yJWv2yLWPJvvVie1c,1071
|
|
30
|
+
agentops_cockpit-0.3.0.dist-info/RECORD,,
|
|
@@ -1,8 +0,0 @@
|
|
|
1
|
-
agent_ops_cockpit/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
2
|
-
agent_ops_cockpit/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
3
|
-
agent_ops_cockpit/cli/main.py,sha256=G_1oEp9xck8knpy5c5q80I1n_9JP7rTheJpd55dJ74M,7624
|
|
4
|
-
agentops_cockpit-0.2.2.dist-info/METADATA,sha256=pnjmdARu64WPpwmmEU-ZAvzZDMFfeXKQbwH3As3NyKA,7246
|
|
5
|
-
agentops_cockpit-0.2.2.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
|
|
6
|
-
agentops_cockpit-0.2.2.dist-info/entry_points.txt,sha256=SOGYPNtUGhMVgxLQ9dEYo7L3M_dvhWEU2eQz2zhaTkY,112
|
|
7
|
-
agentops_cockpit-0.2.2.dist-info/licenses/LICENSE,sha256=XNJEk4bvf88tBnKqHdGBGi10l9yJWv2yLWPJvvVie1c,1071
|
|
8
|
-
agentops_cockpit-0.2.2.dist-info/RECORD,,
|