agentops-cockpit 0.4.1__py3-none-any.whl → 0.9.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agent_ops_cockpit/agent.py +137 -0
- agent_ops_cockpit/cli/main.py +114 -11
- agent_ops_cockpit/eval/load_test.py +15 -10
- agent_ops_cockpit/eval/quality_climber.py +23 -5
- agent_ops_cockpit/eval/red_team.py +16 -10
- agent_ops_cockpit/mcp_server.py +132 -0
- agent_ops_cockpit/ops/arch_review.py +125 -59
- agent_ops_cockpit/ops/cost_optimizer.py +0 -1
- agent_ops_cockpit/ops/evidence_bridge.py +132 -0
- agent_ops_cockpit/ops/frameworks.py +79 -10
- agent_ops_cockpit/ops/mcp_hub.py +1 -2
- agent_ops_cockpit/ops/orchestrator.py +363 -49
- agent_ops_cockpit/ops/pii_scrubber.py +1 -1
- agent_ops_cockpit/ops/policies.json +26 -0
- agent_ops_cockpit/ops/policy_engine.py +85 -0
- agent_ops_cockpit/ops/reliability.py +30 -10
- agent_ops_cockpit/ops/secret_scanner.py +10 -3
- agent_ops_cockpit/ops/ui_auditor.py +91 -96
- agent_ops_cockpit/ops/watcher.py +138 -0
- agent_ops_cockpit/ops/watchlist.json +88 -0
- agent_ops_cockpit/optimizer.py +380 -158
- agent_ops_cockpit/shadow/router.py +7 -8
- agent_ops_cockpit/system_prompt.md +13 -0
- agent_ops_cockpit/tests/golden_set.json +52 -0
- agent_ops_cockpit/tests/test_agent.py +34 -0
- agent_ops_cockpit/tests/test_arch_review.py +45 -0
- agent_ops_cockpit/tests/test_frameworks.py +100 -0
- agent_ops_cockpit/tests/test_optimizer.py +68 -0
- agent_ops_cockpit/tests/test_quality_climber.py +18 -0
- agent_ops_cockpit/tests/test_red_team.py +35 -0
- agent_ops_cockpit/tests/test_secret_scanner.py +24 -0
- agentops_cockpit-0.9.5.dist-info/METADATA +246 -0
- agentops_cockpit-0.9.5.dist-info/RECORD +47 -0
- {agentops_cockpit-0.4.1.dist-info → agentops_cockpit-0.9.5.dist-info}/entry_points.txt +1 -0
- agentops_cockpit-0.4.1.dist-info/METADATA +0 -171
- agentops_cockpit-0.4.1.dist-info/RECORD +0 -31
- {agentops_cockpit-0.4.1.dist-info → agentops_cockpit-0.9.5.dist-info}/WHEEL +0 -0
- {agentops_cockpit-0.4.1.dist-info → agentops_cockpit-0.9.5.dist-info}/licenses/LICENSE +0 -0
agent_ops_cockpit/optimizer.py
CHANGED
|
@@ -1,208 +1,363 @@
|
|
|
1
|
-
import
|
|
1
|
+
from __future__ import annotations
|
|
2
2
|
import os
|
|
3
3
|
import re
|
|
4
|
-
import
|
|
5
|
-
from typing import List, Dict
|
|
4
|
+
from typing import List, Dict, Any
|
|
6
5
|
import typer
|
|
7
6
|
from rich.console import Console
|
|
8
7
|
from rich.table import Table
|
|
9
8
|
from rich.panel import Panel
|
|
10
9
|
from rich.syntax import Syntax
|
|
10
|
+
from packaging import version
|
|
11
|
+
|
|
12
|
+
# Import the evidence bridge
|
|
13
|
+
try:
|
|
14
|
+
from agent_ops_cockpit.ops.evidence_bridge import get_package_evidence, get_compatibility_report
|
|
15
|
+
except ImportError:
|
|
16
|
+
# Fallback for local execution
|
|
17
|
+
try:
|
|
18
|
+
from backend.ops.evidence_bridge import get_package_evidence, get_compatibility_report
|
|
19
|
+
except ImportError:
|
|
20
|
+
# Final fallback
|
|
21
|
+
def get_package_evidence(pkg): return {}
|
|
22
|
+
def get_compatibility_report(imports): return []
|
|
11
23
|
|
|
12
24
|
app = typer.Typer(help="AgentOps Cockpit: The Agent Optimizer CLI")
|
|
13
25
|
console = Console()
|
|
14
26
|
|
|
15
27
|
class OptimizationIssue:
|
|
16
|
-
def __init__(self, id: str, title: str, impact: str, savings: str, description: str, diff: str):
|
|
28
|
+
def __init__(self, id: str, title: str, impact: str, savings: str, description: str, diff: str, package: str = None, fix_pattern: str = None):
|
|
17
29
|
self.id = id
|
|
18
30
|
self.title = title
|
|
19
31
|
self.impact = impact
|
|
20
32
|
self.savings = savings
|
|
21
33
|
self.description = description
|
|
22
34
|
self.diff = diff
|
|
35
|
+
self.package = package
|
|
36
|
+
self.fix_pattern = fix_pattern
|
|
37
|
+
self.evidence = None
|
|
23
38
|
|
|
24
|
-
def analyze_code(content: str, file_path: str = "
|
|
39
|
+
def analyze_code(content: str, file_path: str = "agent.py", versions: Dict[str, str] = None) -> List[OptimizationIssue]:
|
|
25
40
|
issues = []
|
|
26
41
|
content_lower = content.lower()
|
|
42
|
+
versions = versions or {}
|
|
43
|
+
|
|
44
|
+
# --- SITUATIONAL PLATFORM OPTIMIZATIONS ---
|
|
45
|
+
|
|
46
|
+
v_ai = versions.get("google-cloud-aiplatform", "Not Installed")
|
|
47
|
+
if "google.cloud.aiplatform" in content_lower or "vertexai" in content_lower:
|
|
48
|
+
if v_ai == "Not Installed":
|
|
49
|
+
issues.append(OptimizationIssue(
|
|
50
|
+
"vertex_install", "Install Modern Vertex SDK", "HIGH", "90% cost savings",
|
|
51
|
+
"You appear to be using Vertex AI logic but the SDK is not in your environment. Install v1.70.0+ to unlock context caching.",
|
|
52
|
+
"+ # pip install google-cloud-aiplatform>=1.70.0",
|
|
53
|
+
package="google-cloud-aiplatform"
|
|
54
|
+
))
|
|
55
|
+
elif v_ai != "Unknown":
|
|
56
|
+
try:
|
|
57
|
+
if version.parse(v_ai) < version.parse("1.70.0"):
|
|
58
|
+
issues.append(OptimizationIssue(
|
|
59
|
+
"vertex_legacy_opt", "Situational Performance (Legacy SDK)", "MEDIUM", "20% cost savings",
|
|
60
|
+
f"Your SDK ({v_ai}) lacks native Context Caching. Optimize by using selective prompt pruning before execution.",
|
|
61
|
+
"+ from agent_ops_cockpit.ops.cost_optimizer import situational_pruning\n+ pruned = situational_pruning(context)",
|
|
62
|
+
package="google-cloud-aiplatform"
|
|
63
|
+
))
|
|
64
|
+
issues.append(OptimizationIssue(
|
|
65
|
+
"vertex_upgrade_path", "Modernization Path", "HIGH", "90% cost savings",
|
|
66
|
+
"Upgrading to 1.70.0+ enables near-instant token reuse via CachingConfig.",
|
|
67
|
+
"+ # Upgrade to >1.70.0",
|
|
68
|
+
package="google-cloud-aiplatform"
|
|
69
|
+
))
|
|
70
|
+
elif "cache" not in content_lower:
|
|
71
|
+
issues.append(OptimizationIssue(
|
|
72
|
+
"context_caching", "Enable Context Caching", "HIGH", "90% cost reduction",
|
|
73
|
+
"Large model context detected. Use native CachingConfig.",
|
|
74
|
+
"+ cache = vertexai.preview.CachingConfig(ttl=3600)",
|
|
75
|
+
package="google-cloud-aiplatform"
|
|
76
|
+
))
|
|
77
|
+
except Exception:
|
|
78
|
+
pass
|
|
27
79
|
|
|
28
|
-
#
|
|
80
|
+
# OpenAI
|
|
81
|
+
openai_v = versions.get("openai", "Not Installed")
|
|
82
|
+
if "openai" in content_lower:
|
|
83
|
+
if openai_v != "Not Installed" and version.parse(openai_v) < version.parse("1.0.0"):
|
|
84
|
+
issues.append(OptimizationIssue(
|
|
85
|
+
"openai_legacy", "Found Legacy OpenAI SDK", "HIGH", "40% latency reduction",
|
|
86
|
+
f"You are on {openai_v}. Transitioning to the v1.0.0+ Client pattern enables modern streaming and improved error handling.",
|
|
87
|
+
"+ from openai import OpenAI\n+ client = OpenAI()",
|
|
88
|
+
package="openai"
|
|
89
|
+
))
|
|
90
|
+
elif "prompt_cache" not in content_lower:
|
|
91
|
+
issues.append(OptimizationIssue(
|
|
92
|
+
"openai_caching", "OpenAI Prompt Caching", "MEDIUM", "50% latency reduction",
|
|
93
|
+
"OpenAI automatically caches repeated input prefixes. Ensure your system prompt is first.",
|
|
94
|
+
"+ # Ensure system prompt is first\n+ messages = [{'role': 'system', ...}]",
|
|
95
|
+
package="openai"
|
|
96
|
+
))
|
|
29
97
|
|
|
30
|
-
#
|
|
31
|
-
if "
|
|
98
|
+
# Anthropic
|
|
99
|
+
if ("anthropic" in content_lower or "claude" in content_lower) and "orchestra" not in content_lower:
|
|
32
100
|
issues.append(OptimizationIssue(
|
|
33
|
-
"
|
|
34
|
-
"
|
|
35
|
-
"
|
|
36
|
-
"
|
|
37
|
-
"OpenAI automatically caches repeated input prefixes. Ensure your system prompt is at the beginning of the message list.",
|
|
38
|
-
"+ # Ensure system prompt is first and static for optimal caching\n+ messages = [{'role': 'system', 'content': SYSTEM_PROMPT}, ...]"
|
|
101
|
+
"anthropic_orchestration", "Anthropic Orchestration Pattern", "HIGH", "30% reliability boost",
|
|
102
|
+
"Claude performs best with an Orchestrator-Subagent pattern for complex tasks.",
|
|
103
|
+
"+ # Use orchestrator to delegate sub-tasks",
|
|
104
|
+
package="anthropic"
|
|
39
105
|
))
|
|
40
106
|
|
|
41
|
-
#
|
|
42
|
-
if "
|
|
43
|
-
|
|
44
|
-
"
|
|
45
|
-
"
|
|
46
|
-
"
|
|
47
|
-
"
|
|
48
|
-
"Anthropic recommends using an orchestrator to manage subagents for complex tasks. This reduces token spill over and improves reliability.",
|
|
49
|
-
"+ orchestrator = AnthropicOrchestrator()\n+ orchestrator.register_subagent('researcher', researcher_agent)"
|
|
107
|
+
# Microsoft
|
|
108
|
+
if ("autogen" in content_lower or "microsoft" in content_lower) and "workflow" not in content_lower:
|
|
109
|
+
issues.append(OptimizationIssue(
|
|
110
|
+
"ms_workflows", "Microsoft Agent Workflows", "MEDIUM", "40% consistency boost",
|
|
111
|
+
"Using graph-based repeatable workflows ensures enterprise reliability.",
|
|
112
|
+
"+ # Define a repeatable graph-based flow",
|
|
113
|
+
package="pyautogen"
|
|
50
114
|
))
|
|
51
115
|
|
|
52
|
-
#
|
|
53
|
-
if ("
|
|
116
|
+
# AWS
|
|
117
|
+
if ("bedrock" in content_lower or "boto3" in content_lower) and "actiongroup" not in content_lower:
|
|
54
118
|
issues.append(OptimizationIssue(
|
|
55
|
-
"
|
|
56
|
-
"
|
|
57
|
-
"
|
|
58
|
-
"
|
|
59
|
-
"Microsoft best practice: Use the Semantic Kernel Process Framework for stateful, graph-based workflows instead of simple loops.",
|
|
60
|
-
"+ workflow = KernelProcess(name='logic')\n+ workflow.add_step(MyStep())"
|
|
119
|
+
"aws_action_groups", "AWS Bedrock Action Groups", "HIGH", "50% tool reliability",
|
|
120
|
+
"Standardize tool execution via Bedrock Action Group schemas.",
|
|
121
|
+
"+ # Define Bedrock Action Group",
|
|
122
|
+
package="aws-sdk"
|
|
61
123
|
))
|
|
62
124
|
|
|
63
|
-
#
|
|
64
|
-
if "
|
|
125
|
+
# CopilotKit
|
|
126
|
+
if "copilotkit" in content_lower and "usecopilotstate" not in content_lower:
|
|
65
127
|
issues.append(OptimizationIssue(
|
|
66
|
-
"
|
|
67
|
-
"
|
|
68
|
-
"
|
|
69
|
-
"
|
|
70
|
-
"AWS recommends using Action Groups with OpenAPI schemas to manage tool interactions securely and consistently.",
|
|
71
|
-
"+ action_group = bedrock.AgentActionGroup(name='Tools', schema='s3://bucket/api.json')"
|
|
128
|
+
"copilot_state", "CopilotKit Shared State", "MEDIUM", "60% UI responsiveness",
|
|
129
|
+
"Ensure the Face remains aligned with the Engine via shared state sync.",
|
|
130
|
+
"+ # Use shared state for UI alignment",
|
|
131
|
+
package="@copilotkit/react-core"
|
|
72
132
|
))
|
|
73
133
|
|
|
74
|
-
#
|
|
75
|
-
if "
|
|
134
|
+
# Routing
|
|
135
|
+
if "pro" in content_lower and "flash" not in content_lower:
|
|
76
136
|
issues.append(OptimizationIssue(
|
|
77
|
-
"
|
|
78
|
-
"
|
|
79
|
-
"
|
|
80
|
-
"
|
|
81
|
-
"CopilotKit best practice: Use shared state to keep the frontend (Face) and agent (Engine) reactive and aligned.",
|
|
82
|
-
"+ state = useCopilotState({ 'user': user_id })\n+ agent.sync_state(state)"
|
|
137
|
+
"model_routing", "Smart Model Routing", "HIGH", "70% cost savings",
|
|
138
|
+
"Route simple queries to Flash models to minimize consumption.",
|
|
139
|
+
"+ if is_simple(q): model = 'gemini-1.5-flash'",
|
|
140
|
+
package="google-cloud-aiplatform"
|
|
83
141
|
))
|
|
84
142
|
|
|
85
|
-
#
|
|
86
|
-
|
|
87
|
-
# Check for large system instructions
|
|
88
|
-
large_string_pattern = re.compile(r'"""[\s\S]{200,}"""|\'\'\'[\s\S]{200,}\'\'\'')
|
|
89
|
-
if large_string_pattern.search(content) and "cache" not in content_lower:
|
|
143
|
+
# Infrastructure (Cloud Run + GKE)
|
|
144
|
+
if "cloud run" in content_lower and "cpu_boost" not in content_lower:
|
|
90
145
|
issues.append(OptimizationIssue(
|
|
91
|
-
"
|
|
92
|
-
"Enable
|
|
93
|
-
"
|
|
94
|
-
"
|
|
95
|
-
"Large static system instructions detected. Using context caching (Gemini/Anthropic) prevents redundant token processing.",
|
|
96
|
-
"+ cache = vertexai.preview.CachingConfig(ttl=3600)\n+ model = GenerativeModel('gemini-1.5-pro', caching_config=cache)"
|
|
146
|
+
"cr_startup_boost", "Cloud Run Startup Boost", "HIGH", "50% latency reduction",
|
|
147
|
+
"Enable Startup CPU Boost to reduce cold-start latency for Python agents.",
|
|
148
|
+
"+ startup_cpu_boost: true",
|
|
149
|
+
package="google-cloud-run"
|
|
97
150
|
))
|
|
98
|
-
|
|
99
|
-
# Check for hardcoded Pro model usage where Flash might suffice
|
|
100
|
-
if re.search(r"\bpro\b", content_lower) and not any(re.search(rf"\b{w}\b", content_lower) for w in ["flash", "mini", "haiku"]):
|
|
151
|
+
if ("gke" in content_lower or "kubernetes" in content_lower) and "identity" not in content_lower:
|
|
101
152
|
issues.append(OptimizationIssue(
|
|
102
|
-
"
|
|
103
|
-
"
|
|
104
|
-
"
|
|
105
|
-
"
|
|
106
|
-
"Explicit usage of Pro/Opus models detected. Consider Flash (Google), Mini (OpenAI), or Haiku (Anthropic) for non-reasoning tasks.",
|
|
107
|
-
"- model = 'gpt-4o'\n+ model = 'gpt-4o-mini' # Or use model_router"
|
|
153
|
+
"gke_identity", "GKE Workload Identity", "HIGH", "100% security baseline",
|
|
154
|
+
"Use Workload Identity for secure service-to-service communication.",
|
|
155
|
+
"+ # Use Workload Identity",
|
|
156
|
+
package="google-cloud-gke"
|
|
108
157
|
))
|
|
109
158
|
|
|
110
|
-
#
|
|
111
|
-
if "
|
|
112
|
-
|
|
113
|
-
"
|
|
114
|
-
"
|
|
115
|
-
"
|
|
116
|
-
"
|
|
117
|
-
|
|
118
|
-
|
|
159
|
+
# Language Specific (Go + Node)
|
|
160
|
+
if file_path.endswith(".go") and "goroutine" not in content_lower:
|
|
161
|
+
issues.append(OptimizationIssue(
|
|
162
|
+
"go_concurrency", "Go Native Concurrency", "HIGH", "80% throughput boost",
|
|
163
|
+
"Leveraging Goroutines for parallel tool execution is a Go best practice.",
|
|
164
|
+
"+ go func() { tool.execute() }()",
|
|
165
|
+
package="golang"
|
|
166
|
+
))
|
|
167
|
+
if (file_path.endswith(".ts") or file_path.endswith(".js") or "axios" in content_lower) and "fetch" not in content_lower:
|
|
168
|
+
issues.append(OptimizationIssue(
|
|
169
|
+
"node_native_fetch", "Native Fetch API", "MEDIUM", "20% bundle reduction",
|
|
170
|
+
"Node 20+ supports native fetch, reducing dependency on heavy libraries like axios.",
|
|
171
|
+
"+ const res = await fetch(url);",
|
|
172
|
+
package="nodejs"
|
|
119
173
|
))
|
|
120
174
|
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
175
|
+
lg_v = versions.get("langgraph", "Not Installed")
|
|
176
|
+
if "langgraph" in content_lower:
|
|
177
|
+
if lg_v != "Not Installed" and lg_v != "Unknown":
|
|
178
|
+
try:
|
|
179
|
+
if version.parse(lg_v) < version.parse("0.1.0"):
|
|
180
|
+
issues.append(OptimizationIssue(
|
|
181
|
+
"langgraph_legacy", "Situational Stability (Legacy LangGraph)", "HIGH", "Stability Boost",
|
|
182
|
+
f"You are on {lg_v}. Older versions lack the hardened StateGraph compilation. Upgrade is recommended.",
|
|
183
|
+
"+ # Consider upgrading for better persistence",
|
|
184
|
+
package="langgraph"
|
|
185
|
+
))
|
|
186
|
+
except Exception:
|
|
187
|
+
pass
|
|
188
|
+
|
|
189
|
+
if "persistence" not in content_lower and "checkpointer" not in content_lower:
|
|
126
190
|
issues.append(OptimizationIssue(
|
|
127
|
-
"
|
|
128
|
-
"
|
|
129
|
-
"
|
|
130
|
-
"
|
|
131
|
-
"Detected Cloud Run deployment without CPU Boost. Enabling this in your terraform/cloud-run-ui reduces startup latency for Python agents.",
|
|
132
|
-
"+ metadata:\n+ annotations:\n+ run.googleapis.com/startup-cpu-boost: 'true'"
|
|
191
|
+
"langgraph_persistence", "LangGraph Persistence", "HIGH", "100% state recovery",
|
|
192
|
+
"A checkpointer is mandatory for reliable long-running agents.",
|
|
193
|
+
"+ graph.compile(checkpointer=checkpointer)",
|
|
194
|
+
package="langgraph"
|
|
133
195
|
))
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
"
|
|
140
|
-
"GKE Workload Identity Implementation",
|
|
141
|
-
"HIGH",
|
|
142
|
-
"Enhanced Security & Audit",
|
|
143
|
-
"Detected GKE deployment using static keys or default service accounts. Use Workload Identity for least-privilege tool access.",
|
|
144
|
-
"+ iam.gke.io/gcp-service-account: agent-sa@project.iam.gserviceaccount.com"
|
|
196
|
+
if "recursion" not in content_lower:
|
|
197
|
+
issues.append(OptimizationIssue(
|
|
198
|
+
"langgraph_recursion", "Recursion Limits", "MEDIUM", "Safety Guardrail",
|
|
199
|
+
"Set recursion limits to prevent expensive infinite loops in cyclic graphs.",
|
|
200
|
+
"+ graph.invoke(..., config={'recursion_limit': 50})",
|
|
201
|
+
package="langgraph"
|
|
145
202
|
))
|
|
146
203
|
|
|
147
|
-
# ---
|
|
204
|
+
# --- ARCHITECTURAL OPTIMIZATIONS ---
|
|
148
205
|
|
|
149
|
-
#
|
|
150
|
-
|
|
206
|
+
# Large system instructions
|
|
207
|
+
large_string_pattern = re.compile(r'"""[\s\S]{200,}"""|\'\'\'[\s\S]{200,}\'\'\'')
|
|
208
|
+
if large_string_pattern.search(content) and "cache" not in content_lower:
|
|
151
209
|
issues.append(OptimizationIssue(
|
|
152
|
-
"
|
|
153
|
-
"
|
|
154
|
-
"
|
|
155
|
-
"
|
|
156
|
-
"Detected a standard raw map in Go code. For high-concurrency agents, use sync.Map or a Mutex to prevent fatal panics under load.",
|
|
157
|
-
"- state := make(map[string]int)\n+ var state sync.Map"
|
|
210
|
+
"context_caching", "Enable Context Caching", "HIGH", "90% cost reduction",
|
|
211
|
+
"Large static system instructions detected. Use context caching.",
|
|
212
|
+
"+ cache = vertexai.preview.CachingConfig(ttl=3600)",
|
|
213
|
+
package="google-cloud-aiplatform"
|
|
158
214
|
))
|
|
159
215
|
|
|
160
|
-
#
|
|
161
|
-
if
|
|
162
|
-
|
|
163
|
-
"
|
|
164
|
-
"
|
|
165
|
-
"
|
|
166
|
-
"
|
|
167
|
-
"Detected external HTTP libraries. Node 20+ supports high-performance native fetch(), which simplifies the dependency tree.",
|
|
168
|
-
"- import axios from 'axios'\n+ const resp = await fetch(url)"
|
|
216
|
+
# Missing semantic cache
|
|
217
|
+
if "hive_mind" not in content_lower and "cache" not in content_lower:
|
|
218
|
+
issues.append(OptimizationIssue(
|
|
219
|
+
"semantic_caching", "Implement Semantic Caching", "HIGH", "40-60% savings",
|
|
220
|
+
"No caching layer detected. Adding a semantic cache reduces LLM costs.",
|
|
221
|
+
"+ @hive_mind(cache=global_cache)",
|
|
222
|
+
package="google-adk"
|
|
169
223
|
))
|
|
170
224
|
|
|
171
|
-
# ---
|
|
225
|
+
# --- BEST PRACTICE OPTIMIZATIONS ---
|
|
172
226
|
|
|
173
|
-
#
|
|
174
|
-
if
|
|
227
|
+
# Prompt Externalization
|
|
228
|
+
if large_string_pattern.search(content):
|
|
175
229
|
issues.append(OptimizationIssue(
|
|
176
|
-
"
|
|
177
|
-
"
|
|
178
|
-
"
|
|
179
|
-
"Cross-session memory & Safety",
|
|
180
|
-
"Detected LangGraph usage without a checkpointer. Persistence is mandatory for production agents to resume from failures or maintain long-term state.",
|
|
181
|
-
"+ checkpointer = MemorySaver()\n+ app = workflow.compile(checkpointer=checkpointer)"
|
|
230
|
+
"external_prompts", "Externalize System Prompts", "MEDIUM", "Architectural Debt Reduction",
|
|
231
|
+
"Keeping large system prompts in code makes them hard to version and test. Move them to 'system_prompt.md' and load dynamically.",
|
|
232
|
+
"+ with open('system_prompt.md', 'r') as f:\n+ SYSTEM_PROMPT = f.read()"
|
|
182
233
|
))
|
|
183
234
|
|
|
184
|
-
#
|
|
185
|
-
if "
|
|
235
|
+
# Resiliency / Retries
|
|
236
|
+
if "retry" not in content_lower and "tenacity" not in content_lower:
|
|
186
237
|
issues.append(OptimizationIssue(
|
|
187
|
-
"
|
|
188
|
-
"
|
|
189
|
-
"
|
|
190
|
-
"
|
|
191
|
-
"No recursion limit detected in LangGraph config. Setting a limit (e.g., 50) prevents infinite loops in cyclic agent graphs.",
|
|
192
|
-
"+ config = {'recursion_limit': 50}\n+ app.invoke(inputs, config)"
|
|
238
|
+
"resiliency_retries", "Implement Exponential Backoff", "HIGH", "99.9% Reliability",
|
|
239
|
+
"Your agent calls external APIs/DBs but has no retry logic. Use 'tenacity' to handle transient failures.",
|
|
240
|
+
"+ @retry(wait=wait_exponential(multiplier=1, min=4, max=10), stop=stop_after_attempt(3))",
|
|
241
|
+
package="tenacity"
|
|
193
242
|
))
|
|
194
243
|
|
|
244
|
+
# Session Management
|
|
245
|
+
if "session" not in content_lower and "conversation_id" not in content_lower:
|
|
246
|
+
issues.append(OptimizationIssue(
|
|
247
|
+
"session_management", "Add Session Tracking", "MEDIUM", "User Continuity",
|
|
248
|
+
"No session tracking detected. Agents in production need a 'conversation_id' to maintain multi-turn context.",
|
|
249
|
+
"+ def chat(q: str, conversation_id: str = None):"
|
|
250
|
+
))
|
|
251
|
+
|
|
252
|
+
# Pinecone Optimization
|
|
253
|
+
if "pinecone" in content_lower:
|
|
254
|
+
if "grpc" not in content_lower:
|
|
255
|
+
issues.append(OptimizationIssue(
|
|
256
|
+
"pinecone_grpc", "Pinecone High-Perf (gRPC)", "MEDIUM", "40% latency reduction",
|
|
257
|
+
"You are using the standard Pinecone client. Switching to pinecone[grpc] enables low-latency streaming for large vector retrievals.",
|
|
258
|
+
"+ from pinecone.grpc import PineconeGRPC as Pinecone\n+ pc = Pinecone(api_key='...')"
|
|
259
|
+
))
|
|
260
|
+
if "namespace" not in content_lower:
|
|
261
|
+
issues.append(OptimizationIssue(
|
|
262
|
+
"pinecone_isolation", "Pinecone Namespace Isolation", "MEDIUM", "RAG Accuracy Boost",
|
|
263
|
+
"No namespaces detected. Use namespaces to isolate user data or document segments for more accurate retrieval.",
|
|
264
|
+
"+ index.query(..., namespace='customer-a')"
|
|
265
|
+
))
|
|
266
|
+
|
|
267
|
+
# Google Cloud Database Optimizations
|
|
268
|
+
|
|
269
|
+
# AlloyDB
|
|
270
|
+
if "alloydb" in content_lower:
|
|
271
|
+
if "columnar" not in content_lower:
|
|
272
|
+
issues.append(OptimizationIssue(
|
|
273
|
+
"alloydb_columnar", "AlloyDB Columnar Engine", "HIGH", "100x Query Speedup",
|
|
274
|
+
"AlloyDB detected. Enable the Columnar Engine for analytical and AI-driven vector queries.",
|
|
275
|
+
"+ # Enable AlloyDB Columnar Engine for vector scaling"
|
|
276
|
+
))
|
|
277
|
+
|
|
278
|
+
# BigQuery
|
|
279
|
+
if "bigquery" in content_lower or "bq" in content_lower:
|
|
280
|
+
if "vector_search" not in content_lower:
|
|
281
|
+
issues.append(OptimizationIssue(
|
|
282
|
+
"bq_vector_search", "BigQuery Vector Search", "HIGH", "FinOps: Serverless RAG",
|
|
283
|
+
"BigQuery detected. Use BQ Vector Search for cost-effective RAG over massive datasets without moving data to a separate DB.",
|
|
284
|
+
"+ SELECT * FROM VECTOR_SEARCH(TABLE my_dataset.embeddings, ...)"
|
|
285
|
+
))
|
|
286
|
+
|
|
287
|
+
# Cloud SQL
|
|
288
|
+
if "cloudsql" in content_lower or "psycopg2" in content_lower or "sqlalchemy" in content_lower:
|
|
289
|
+
if "cloud-sql-connector" not in content_lower:
|
|
290
|
+
issues.append(OptimizationIssue(
|
|
291
|
+
"cloudsql_connector", "Cloud SQL Python Connector", "MEDIUM", "100% Secure Auth",
|
|
292
|
+
"Using raw drivers detected. Use the official Cloud SQL Python Connector for IAM-based authentication and automatic encryption.",
|
|
293
|
+
"+ from google.cloud.sql.connector import Connector\n+ connector = Connector()"
|
|
294
|
+
))
|
|
295
|
+
|
|
296
|
+
# Firestore
|
|
297
|
+
if "firestore" in content_lower:
|
|
298
|
+
if "vector" not in content_lower:
|
|
299
|
+
issues.append(OptimizationIssue(
|
|
300
|
+
"firestore_vector", "Firestore Vector Search (Native)", "HIGH", "Real-time RAG",
|
|
301
|
+
"Firestore detected. Use native Vector Search and KNN queries for high-concurrency mobile/web agent retrieval.",
|
|
302
|
+
"+ collection.find_nearest(vector_field='embedding', ...)"
|
|
303
|
+
))
|
|
304
|
+
|
|
305
|
+
# Oracle OCI Optimizations
|
|
306
|
+
if "oci" in content_lower or "oracle" in content_lower:
|
|
307
|
+
if "resource_principal" not in content_lower:
|
|
308
|
+
issues.append(OptimizationIssue(
|
|
309
|
+
"oci_auth", "OCI Resource Principals", "HIGH", "100% Secure Auth",
|
|
310
|
+
"Using static config/keys detected on OCI. Use Resource Principals for secure, credential-less access from OCI compute.",
|
|
311
|
+
"+ auth = oci.auth.signers.get_resource_principals_signer()"
|
|
312
|
+
))
|
|
313
|
+
|
|
314
|
+
# CrewAI Optimizations
|
|
315
|
+
if "crewai" in content_lower or "crew(" in content_lower:
|
|
316
|
+
if "manager_agent" not in content_lower and "hierarchical" not in content_lower:
|
|
317
|
+
issues.append(OptimizationIssue(
|
|
318
|
+
"crewai_manager", "Use Hierarchical Manager", "MEDIUM", "30% Coordination Boost",
|
|
319
|
+
"Your crew uses sequential execution. For complex tasks, a Manager Agent improves task handoffs and reasoning.",
|
|
320
|
+
"+ crew = Crew(..., process=Process.hierarchical, manager_agent=manager)"
|
|
321
|
+
))
|
|
322
|
+
|
|
195
323
|
return issues
|
|
196
324
|
|
|
325
|
+
def estimate_savings(token_count: int, issues: List[OptimizationIssue]) -> Dict[str, Any]:
|
|
326
|
+
baseline_cost_per_m = 10.0
|
|
327
|
+
monthly_requests = 10000
|
|
328
|
+
current_cost = (token_count / 1_000_000) * baseline_cost_per_m * monthly_requests
|
|
329
|
+
|
|
330
|
+
total_savings_pct = 0.0
|
|
331
|
+
for issue in issues:
|
|
332
|
+
if "90%" in issue.savings:
|
|
333
|
+
total_savings_pct += 0.45 # Context Caching / Modern SDK
|
|
334
|
+
elif "70%" in issue.savings:
|
|
335
|
+
total_savings_pct += 0.35 # Smart Routing (Pro -> Flash)
|
|
336
|
+
elif "50%" in issue.savings:
|
|
337
|
+
total_savings_pct += 0.20 # Infrastructure / Startup Boost
|
|
338
|
+
elif "40-60%" in issue.savings:
|
|
339
|
+
total_savings_pct += 0.25 # Semantic Caching (Hive Mind)
|
|
340
|
+
else:
|
|
341
|
+
total_savings_pct += 0.05 # Standard Best Practices
|
|
342
|
+
|
|
343
|
+
projected_savings = current_cost * min(total_savings_pct, 0.85)
|
|
344
|
+
|
|
345
|
+
return {
|
|
346
|
+
"current_monthly": current_cost,
|
|
347
|
+
"projected_savings": projected_savings,
|
|
348
|
+
"new_monthly": current_cost - projected_savings
|
|
349
|
+
}
|
|
350
|
+
|
|
197
351
|
@app.command()
|
|
198
352
|
def audit(
|
|
199
353
|
file_path: str = typer.Argument("agent.py", help="Path to the agent code to audit"),
|
|
200
|
-
interactive: bool = typer.Option(True, "--interactive/--no-interactive", "-i", help="Run in interactive mode")
|
|
354
|
+
interactive: bool = typer.Option(True, "--interactive/--no-interactive", "-i", help="Run in interactive mode"),
|
|
355
|
+
apply_fix: bool = typer.Option(False, "--apply", "--fix", help="Automatically apply recommended fixes"),
|
|
356
|
+
quick: bool = typer.Option(False, "--quick", "-q", help="Skip live evidence fetching for faster execution")
|
|
201
357
|
):
|
|
202
|
-
"""
|
|
203
|
-
Audits agent code and proposes cost/perf optimizations.
|
|
204
|
-
"""
|
|
205
358
|
console.print(Panel.fit("🔍 [bold blue]GCP AGENT OPS: OPTIMIZER AUDIT[/bold blue]", border_style="blue"))
|
|
359
|
+
if quick:
|
|
360
|
+
console.print("[dim]⚡ Running in Quick Mode (skipping live evidence fetches)[/dim]")
|
|
206
361
|
console.print(f"Target: [yellow]{file_path}[/yellow]")
|
|
207
362
|
|
|
208
363
|
if not os.path.exists(file_path):
|
|
@@ -212,52 +367,119 @@ def audit(
|
|
|
212
367
|
with open(file_path, 'r') as f:
|
|
213
368
|
content = f.read()
|
|
214
369
|
|
|
370
|
+
# Heuristic: Find all imported packages
|
|
371
|
+
imports = re.findall(r"(?:from|import)\s+([\w\.-]+)", content)
|
|
372
|
+
|
|
373
|
+
from agent_ops_cockpit.ops.evidence_bridge import get_installed_version
|
|
374
|
+
package_versions = { pkg: get_installed_version(pkg) for pkg in ["google-cloud-aiplatform", "openai", "anthropic", "langgraph", "crewai"] }
|
|
375
|
+
|
|
215
376
|
token_estimate = len(content.split()) * 1.5
|
|
216
377
|
console.print(f"📊 Token Metrics: ~[bold]{token_estimate:.0f}[/bold] prompt tokens detected.")
|
|
217
378
|
|
|
218
|
-
|
|
219
|
-
|
|
379
|
+
issues = analyze_code(content, file_path, versions=package_versions)
|
|
380
|
+
# Inject live evidence (skip in quick mode)
|
|
381
|
+
if not quick:
|
|
382
|
+
for issue in issues:
|
|
383
|
+
if issue.package:
|
|
384
|
+
issue.evidence = get_package_evidence(issue.package)
|
|
220
385
|
|
|
221
|
-
|
|
222
|
-
|
|
386
|
+
# --- CROSS-PACKAGE VALIDATION ---
|
|
387
|
+
comp_reports = get_compatibility_report(imports)
|
|
388
|
+
|
|
389
|
+
if comp_reports:
|
|
390
|
+
console.print("\n[bold yellow]🧩 Cross-Package Validation:[/bold yellow]")
|
|
391
|
+
for report in comp_reports:
|
|
392
|
+
if report["type"] == "INCOMPATIBLE":
|
|
393
|
+
console.print(f"❌ [bold red]Conflict Detected:[/bold red] {report['component']} + {report['conflict_with']}")
|
|
394
|
+
console.print(f" [dim]{report['reason']}[/dim]")
|
|
395
|
+
elif report["type"] == "SYNERGY":
|
|
396
|
+
console.print(f"✅ [bold green]Synergy Verified:[/bold green] {report['component']} is optimally paired.")
|
|
223
397
|
|
|
224
398
|
if not issues:
|
|
225
|
-
console.print("\n[bold green]✅ No immediate
|
|
226
|
-
|
|
399
|
+
console.print("\n[bold green]✅ No immediate code-level optimizations found. Your agent is lean![/bold green]")
|
|
400
|
+
if not comp_reports:
|
|
401
|
+
return
|
|
402
|
+
else:
|
|
403
|
+
raise typer.Exit(0)
|
|
404
|
+
|
|
405
|
+
savings = estimate_savings(token_estimate, issues)
|
|
406
|
+
finops_panel = Panel(
|
|
407
|
+
f"💰 [bold]FinOps Projection (Est. 10k req/mo)[/bold]\n"
|
|
408
|
+
f"Current Monthly Spend: [red]${savings['current_monthly']:.2f}[/red]\n"
|
|
409
|
+
f"Projected Savings: [green]${savings['projected_savings']:.2f}[/green]\n"
|
|
410
|
+
f"New Monthly Spend: [blue]${savings['new_monthly']:.2f}[/blue]",
|
|
411
|
+
title="[bold yellow]Financial Optimization[/bold yellow]",
|
|
412
|
+
border_style="yellow"
|
|
413
|
+
)
|
|
414
|
+
console.print(finops_panel)
|
|
227
415
|
|
|
228
416
|
applied = 0
|
|
229
417
|
rejected = 0
|
|
418
|
+
fixed_content = content
|
|
230
419
|
|
|
231
420
|
for opt in issues:
|
|
232
421
|
console.print(f"\n[bold white on blue] --- [{opt.impact} IMPACT] {opt.title} --- [/bold white on blue]")
|
|
233
422
|
console.print(f"Benefit: [green]{opt.savings}[/green]")
|
|
234
423
|
console.print(f"Reason: {opt.description}")
|
|
235
|
-
|
|
424
|
+
|
|
425
|
+
if opt.evidence and "error" not in opt.evidence:
|
|
426
|
+
ev = opt.evidence
|
|
427
|
+
ev_title = "[dim]SDK Citation & Evidence[/dim]"
|
|
428
|
+
|
|
429
|
+
# Highlight if an upgrade is required for maximum efficiency
|
|
430
|
+
if ev.get("upgrade_required"):
|
|
431
|
+
console.print("🚨 [bold yellow]URGENT UPGRADE RECOMMENDED[/bold yellow]")
|
|
432
|
+
console.print(f" Current: {ev['installed_version']} | Required for optimization: >={ev['min_optimized_version']}")
|
|
433
|
+
ev_title = "[bold red]UPGRADE REQUIRED Evidence[/bold red]"
|
|
434
|
+
|
|
435
|
+
ev_panel = Panel(
|
|
436
|
+
f"🔗 [bold]Source[/bold]: {ev['source_url']}\n"
|
|
437
|
+
f"📅 [bold]Latest Release[/bold]: {ev['release_date'][:10]}\n"
|
|
438
|
+
f"📝 [bold]Note[/bold]: {ev['best_practice_context']}",
|
|
439
|
+
title=ev_title,
|
|
440
|
+
border_style="red" if ev.get("upgrade_required") else "dim"
|
|
441
|
+
)
|
|
442
|
+
console.print(ev_panel)
|
|
443
|
+
# Orchestrator parsing
|
|
444
|
+
console.print(f"SOURCE: {opt.title} | {ev['source_url']} | {ev['best_practice_context'].replace('\\n', ' ')}")
|
|
445
|
+
|
|
236
446
|
syntax = Syntax(opt.diff, "python", theme="monokai", line_numbers=False)
|
|
237
447
|
console.print(syntax)
|
|
238
448
|
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
449
|
+
# Output ACTION: for report generation
|
|
450
|
+
console.print(f"ACTION: {file_path}:1 | Optimization: {opt.title} | {opt.description} (Est. {opt.savings})")
|
|
451
|
+
|
|
452
|
+
do_apply = False
|
|
453
|
+
if apply_fix:
|
|
454
|
+
do_apply = True
|
|
455
|
+
elif interactive:
|
|
456
|
+
do_apply = typer.confirm("\nDo you want to apply this code-level optimization?", default=True)
|
|
457
|
+
|
|
458
|
+
if do_apply:
|
|
459
|
+
console.print("✅ [APPROVED] applying fix...")
|
|
460
|
+
if opt.fix_pattern:
|
|
461
|
+
fixed_content = opt.fix_pattern + fixed_content
|
|
462
|
+
applied += 1
|
|
247
463
|
else:
|
|
248
|
-
console.print("
|
|
464
|
+
console.print("❌ [REJECTED] skipping optimization.")
|
|
465
|
+
rejected += 1
|
|
249
466
|
|
|
467
|
+
if applied > 0:
|
|
468
|
+
with open(file_path, 'w') as f:
|
|
469
|
+
f.write(fixed_content)
|
|
470
|
+
console.print(f"\n✨ [bold green]Applied {applied} optimizations to {file_path}![/bold green]")
|
|
471
|
+
|
|
250
472
|
summary_table = Table(title="🎯 AUDIT SUMMARY")
|
|
251
473
|
summary_table.add_column("Category", style="cyan")
|
|
252
474
|
summary_table.add_column("Count", style="magenta")
|
|
253
475
|
summary_table.add_row("Optimizations Applied", str(applied))
|
|
254
476
|
summary_table.add_row("Optimizations Rejected", str(rejected))
|
|
255
477
|
console.print(summary_table)
|
|
256
|
-
|
|
257
|
-
if
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
478
|
+
|
|
479
|
+
# CI/CD Enforcement: Fail if high-impact issues remain in non-interactive mode
|
|
480
|
+
if not interactive and any(opt.impact == "HIGH" for opt in issues):
|
|
481
|
+
console.print("\n[bold red]❌ HIGH IMPACT issues detected. Optimization required for production.[/bold red]")
|
|
482
|
+
raise typer.Exit(code=1)
|
|
261
483
|
|
|
262
484
|
if __name__ == "__main__":
|
|
263
485
|
app()
|