agentops-cockpit 0.4.1__py3-none-any.whl → 0.9.5__py3-none-any.whl

agent_ops_cockpit/shadow/router.py

@@ -1,7 +1,8 @@
 import asyncio
 import json
 import uuid
-from typing import Dict, Any, Callable
+import os
+from typing import Any, Callable
 from datetime import datetime
 
 class ShadowRouter:
@@ -13,17 +14,17 @@ class ShadowRouter:
         self.v1 = v1_func
         self.v2 = v2_func
 
-    async def route(self, query: str, context: Dict[str, Any] = None):
+    async def route(self, query: str, **kwargs):
         trace_id = str(uuid.uuid4())
         
         # 1. Primary Call (Production v1) - Sequential/Blocking
         start_v1 = datetime.now()
-        v1_resp = await self.v1(query, context)
+        v1_resp = await self.v1(query, **kwargs)
         v1_latency = (datetime.now() - start_v1).total_seconds()
 
         # 2. Shadow Call (Experimental v2) - Asynchronous/Non-blocking
         # We fire and forget this, or use a background task
-        asyncio.create_task(self._run_shadow(trace_id, query, context, v1_resp, v1_latency))
+        asyncio.create_task(self._run_shadow(trace_id, query, v1_resp, v1_latency, **kwargs))
 
         return {
             "response": v1_resp,
@@ -31,14 +32,14 @@ class ShadowRouter:
             "latency": v1_latency
         }
 
-    async def _run_shadow(self, trace_id: str, query: str, context: Dict[str, Any], v1_resp: Any, v1_latency: float):
+    async def _run_shadow(self, trace_id: str, query: str, v1_resp: Any, v1_latency: float, **kwargs):
         """
         Runs the v2 agent in the 'shadow' without user impact.
         Logs the comparison to BigQuery/Cloud Logging.
         """
         try:
             start_v2 = datetime.now()
-            v2_resp = await self.v2(query, context)
+            v2_resp = await self.v2(query, **kwargs)
             v2_latency = (datetime.now() - start_v2).total_seconds()
 
             comparison = {
@@ -71,5 +72,3 @@ class ShadowRouter:
         os.makedirs("traces", exist_ok=True)
         with open(f"traces/{data['traceId']}.json", "w") as f:
             json.dump(data, f)
-
-import os

agent_ops_cockpit/system_prompt.md

@@ -0,0 +1,13 @@
+# 🕹️ AgentOps Cockpit: System Persona
+
+You are a professional **Google Well-Architected Agent Orchestrator**. 
+Your primary goal is to assist users in building, optimizing, and securing AI agents on Google Cloud.
+
+## 🛡️ Core Directives:
+1. **Safety First**: Always check for PII leakage and prompt injection before executing logic.
+2. **Operations-Aware**: Frame your responses within the context of the Engine, Face, and Cockpit.
+3. **Structured Recovery**: If a tool fails, suggest a retry strategy with exponential backoff.
+4. **Efficiency**: Use semantic caching whenever possible to reduce token overhead.
+
+## 📡 Output Standard:
+Follow the **A2UI Protocol**. Always return structured JSON that the Face can render.

agent_ops_cockpit/tests/golden_set.json

@@ -0,0 +1,52 @@
+[
+  {"query": "How do I deploy to Google Cloud Run?", "expected": "deploy"},
+  {"query": "What is the A2UI protocol?", "expected": "a2ui"},
+  {"query": "How do I check Hive Mind status?", "expected": "hive mind"},
+  {"query": "Run a security audit on my agent", "expected": "audit"},
+  {"query": "What is the cost of 1M tokens?", "expected": "cost"},
+  {"query": "How to enable context caching?", "expected": "caching"},
+  {"query": "Scan my code for secrets", "expected": "secret"},
+  {"query": "Is my agent well-architected?", "expected": "architecture"},
+  {"query": "Explain shadow routing", "expected": "shadow"},
+  {"query": "Deploy to GKE Autopilot", "expected": "gke"},
+  {"query": "What is a PII scrubber?", "expected": "pii"},
+  {"query": "How to fix prompt injection?", "expected": "injection"},
+  {"query": "Run the red team evaluation", "expected": "red team"},
+  {"query": "Optimize my LLM spend", "expected": "optimize"},
+  {"query": "What are StatBars in A2UI?", "expected": "statbar"},
+  {"query": "How to use the MCP server?", "expected": "mcp"},
+  {"query": "Explain Quality Hill Climbing", "expected": "quality"},
+  {"query": "Check system health", "expected": "health"},
+  {"query": "How to redact credit card numbers?", "expected": "redact"},
+  {"query": "What is the Agentic Trinity?", "expected": "trinity"},
+  {"query": "Setting up Firebase Hosting", "expected": "firebase"},
+  {"query": "How to use the ADK?", "expected": "adk"},
+  {"query": "Detecting hardcoded API keys", "expected": "key"},
+  {"query": "Show me the performance metrics", "expected": "metrics"},
+  {"query": "How to configure VPC Service Controls?", "expected": "vpc"},
+  {"query": "What is the Conflict Guard?", "expected": "conflict"},
+  {"query": "Explain Model Armor integration", "expected": "model armor"},
+  {"query": "How to limit prompt length?", "expected": "limit"},
+  {"query": "Setting up a custom domain", "expected": "domain"},
+  {"query": "How to use structured outputs?", "expected": "structured"},
+  {"query": "What is the cockpit final report?", "expected": "report"},
+  {"query": "How to run a load test?", "expected": "load test"},
+  {"query": "Explain p90 latency", "expected": "p90"},
+  {"query": "How to use the face auditor?", "expected": "ui"},
+  {"query": "Setting up multi-agent swarms", "expected": "multi-agent"},
+  {"query": "What is the situational auditor?", "expected": "situational"},
+  {"query": "How to enable dynamic routing?", "expected": "routing"},
+  {"query": "Explain the regression golden set", "expected": "regression"},
+  {"query": "How to use the Google SDK?", "expected": "sdk"},
+  {"query": "What is the mission control dashboard?", "expected": "dashboard"},
+  {"query": "How to handle token overflow?", "expected": "token"},
+  {"query": "Explain the adversarial attack suite", "expected": "adversarial"},
+  {"query": "How to use workload identity?", "expected": "identity"},
+  {"query": "What is the response match metric?", "expected": "match"},
+  {"query": "How to conduct a design review?", "expected": "review"},
+  {"query": "Explain the FinOps pillar", "expected": "finops"},
+  {"query": "How to use Gemini 1.5 Flash?", "expected": "flash"},
+  {"query": "What is the difference between quick and deep audit?", "expected": "audit"},
+  {"query": "How to setup a checkpointer in LangGraph?", "expected": "checkpointer"},
+  {"query": "Explain the cockpit orchestrator", "expected": "orchestrator"}
+]

agent_ops_cockpit/tests/test_agent.py

@@ -0,0 +1,34 @@
+import os
+import json
+import pytest
+from agent_ops_cockpit.agent import agent_v1_logic
+
+def load_golden_set():
+    path = os.path.join(os.path.dirname(__file__), "golden_set.json")
+    if not os.path.exists(path):
+        return []
+    with open(path, "r") as f:
+        data = json.load(f)
+    return [(item["query"], item["expected"]) for item in data]
+
+@pytest.mark.asyncio
+async def test_agent_v1_logic():
+    """Ensure the agent v1 logic returns a surface."""
+    result = await agent_v1_logic("test query")
+    assert result is not None
+    assert result.surfaceId == "dynamic-response"
+
+def test_well_architected_middlewares():
+    """Verify that core AgentOps middlewares are loaded."""
+    # This is a structural test, asserting true for now as a placeholder
+    assert True 
+
+@pytest.mark.parametrize("query,expected_keyword", load_golden_set())
+@pytest.mark.asyncio
+async def test_regression_golden_set(query, expected_keyword):
+    """Regression suite: Ensure core queries always return relevant keywords."""
+    # In a real test, we would mock the LLM or check local logic
+    # Here we simulate the logic being tested
+    await agent_v1_logic(query)
+    # Simple heuristic check for the demonstration
+    assert True 

agent_ops_cockpit/tests/test_arch_review.py

@@ -0,0 +1,45 @@
+from typer.testing import CliRunner
+from agent_ops_cockpit.ops.arch_review import app
+
+runner = CliRunner()
+
+def test_arch_review_score(tmp_path):
+    # Set up a mock project directory
+    project_dir = tmp_path / "my_agent"
+    project_dir.mkdir()
+    
+    # Create a README to trigger a framework (e.g., Google)
+    readme = project_dir / "README.md"
+    readme.write_text("Uses Google Cloud and Vertex AI.")
+    
+    # Create a code file with some keywords to pass checks
+    code_file = project_dir / "agent.py"
+    code_file.write_text("""
+def chat():
+    # pii scrubbing
+    text = scrub_pii(input)
+    # cache enabled
+    cache = redis.Cache()
+    # iam auth
+    auth = iam.Auth()
+""")
+    
+    # Run the audit on the mock project directory
+    # We need to ensure src is in PYTHONPATH if the test runner doesn't handle it
+    # But usually, when running pytest from root, 'src' is handled or we rely on the import path
+    
+    result = runner.invoke(app, ["--path", str(project_dir)])
+    assert result.exit_code == 0
+    assert "ARCHITECTURE REVIEW" in result.stdout
+    assert "Review Score:" in result.stdout
+    # We expect some checks to pass because of the keywords
+    assert "PASSED" in result.stdout
+
+def test_arch_review_fail_on_empty(tmp_path):
+    project_dir = tmp_path / "empty_agent"
+    project_dir.mkdir()
+    
+    result = runner.invoke(app, ["--path", str(project_dir)])
+    assert result.exit_code == 0
+    assert "FAIL" in result.stdout
+    assert "Review Score: 0/100" in result.stdout

agent_ops_cockpit/tests/test_frameworks.py

@@ -0,0 +1,100 @@
+from agent_ops_cockpit.ops.frameworks import detect_framework
+
+def test_detect_google_framework(tmp_path):
+    # Create a mock README with Google indicators
+    d = tmp_path / "google_project"
+    d.mkdir()
+    readme = d / "README.md"
+    readme.write_text("This project uses Vertex AI and ADK.")
+    
+    assert detect_framework(str(d)) == "google"
+
+def test_detect_openai_framework(tmp_path):
+    d = tmp_path / "openai_project"
+    d.mkdir()
+    reqs = d / "requirements.txt"
+    reqs.write_text("openai>=1.0.0\nlangchain")
+    
+    assert detect_framework(str(d)) == "openai"
+
+def test_detect_anthropic_framework(tmp_path):
+    d = tmp_path / "anthropic_project"
+    d.mkdir()
+    readme = d / "README.md"
+    readme.write_text("Powered by Anthropic Claude 3.5 Sonnet.")
+    
+    assert detect_framework(str(d)) == "anthropic"
+
+def test_detect_microsoft_framework(tmp_path):
+    d = tmp_path / "ms_project"
+    d.mkdir()
+    readme = d / "README.md"
+    readme.write_text("Multi-agent system built with AutoGen.")
+    
+    assert detect_framework(str(d)) == "microsoft"
+
+def test_detect_aws_framework(tmp_path):
+    d = tmp_path / "aws_project"
+    d.mkdir()
+    reqs = d / "requirements.txt"
+    reqs.write_text("boto3\naws-sdk")
+    
+    assert detect_framework(str(d)) == "aws"
+
+def test_detect_copilotkit_framework(tmp_path):
+    d = tmp_path / "copilot_project"
+    d.mkdir()
+    readme = d / "README.md"
+    readme.write_text("Integrated using CopilotKit.ai sidebar.")
+    
+    assert detect_framework(str(d)) == "copilotkit"
+
+def test_detect_generic_framework(tmp_path):
+    d = tmp_path / "generic_project"
+    d.mkdir()
+    readme = d / "README.md"
+    readme.write_text("A simple python script.")
+    
+    assert detect_framework(str(d)) == "generic"
+
+def test_detect_go_framework(tmp_path):
+    d = tmp_path / "go_project"
+    d.mkdir()
+    mod = d / "go.mod"
+    mod.write_text("module agent-go\ngo 1.21")
+    assert detect_framework(str(d)) == "go"
+
+def test_detect_nodejs_framework(tmp_path):
+    d = tmp_path / "node_project"
+    d.mkdir()
+    pkg = d / "package.json"
+    pkg.write_text('{"name": "agent-node"}')
+    assert detect_framework(str(d)) == "nodejs"
+
+def test_detect_streamlit_framework(tmp_path):
+    d = tmp_path / "streamlit_project"
+    d.mkdir()
+    readme = d / "README.md"
+    readme.write_text("Uses streamlit for the UI.")
+    assert detect_framework(str(d)) == "streamlit"
+
+def test_detect_lit_framework(tmp_path):
+    d = tmp_path / "lit_project"
+    d.mkdir()
+    readme = d / "README.md"
+    readme.write_text("Web components with lit-element.")
+    assert detect_framework(str(d)) == "lit"
+
+def test_detect_angular_framework(tmp_path):
+    d = tmp_path / "angular_project"
+    d.mkdir()
+    readme = d / "README.md"
+    readme.write_text("Enterprise agent with @angular/core.")
+    assert detect_framework(str(d)) == "angular"
+
+def test_detect_firebase_framework(tmp_path):
+    d = tmp_path / "firebase_project"
+    d.mkdir()
+    fb = d / "firebase.json"
+    fb.write_text("{}")
+    assert detect_framework(str(d)) == "firebase"

agent_ops_cockpit/tests/test_optimizer.py

@@ -0,0 +1,68 @@
+from agent_ops_cockpit.optimizer import analyze_code
+
+def test_analyze_openai_missing_cache():
+    code = "import openai\nclient = openai.OpenAI()"
+    issues = analyze_code(code)
+    assert any(issue.id == "openai_caching" for issue in issues)
+
+def test_analyze_anthropic_missing_orchestrator():
+    code = "import anthropic\nclient = anthropic.Anthropic()"
+    issues = analyze_code(code)
+    assert any(issue.id == "anthropic_orchestration" for issue in issues)
+
+def test_analyze_microsoft_missing_workflow():
+    code = "from autogen import UserProxyAgent, AssistantAgent"
+    issues = analyze_code(code)
+    assert any(issue.id == "ms_workflows" for issue in issues)
+
+def test_analyze_aws_missing_action_groups():
+    code = "import boto3\nbedrock = boto3.client('bedrock-agent-runtime')"
+    issues = analyze_code(code)
+    assert any(issue.id == "aws_action_groups" for issue in issues)
+
+def test_analyze_copilotkit_missing_shared_state():
+    code = "import copilotkit\n# Some logic without state sync"
+    issues = analyze_code(code)
+    assert any(issue.id == "copilot_state" for issue in issues)
+
+def test_analyze_model_routing_pro_only():
+    code = "model = 'gemini-1.5-pro'"
+    issues = analyze_code(code)
+    assert any(issue.id == "model_routing" for issue in issues)
+
+def test_analyze_missing_semantic_cache():
+    code = "def chat(): pass"
+    issues = analyze_code(code)
+    assert any(issue.id == "semantic_caching" for issue in issues)
+
+def test_analyze_context_caching():
+    code = '"""' + "A" * 300 + '"""'
+    issues = analyze_code(code)
+    assert any(issue.id == "context_caching" for issue in issues)
+
+def test_analyze_infrastructure_optimizations():
+    # Cloud Run
+    cr_code = "# Running on Cloud Run"
+    cr_issues = analyze_code(cr_code)
+    assert any(issue.id == "cr_startup_boost" for issue in cr_issues)
+    
+    # GKE
+    gke_code = "# Running on GKE with Kubernetes"
+    gke_issues = analyze_code(gke_code)
+    assert any(issue.id == "gke_identity" for issue in gke_issues)
+
+def test_analyze_language_optimizations():
+    # Go
+    go_code = "state := make(map[string]int)"
+    go_issues = analyze_code(go_code, "main.go")
+    assert any(issue.id == "go_concurrency" for issue in go_issues)
+    
+    # NodeJS
+    js_code = "import axios from 'axios'"
+    js_issues = analyze_code(js_code, "app.ts")
+    assert any(issue.id == "node_native_fetch" for issue in js_issues)
+def test_analyze_langgraph_optimizations():
+    code = "from langgraph.graph import StateGraph"
+    issues = analyze_code(code)
+    assert any(issue.id == "langgraph_persistence" for issue in issues)
+    assert any(issue.id == "langgraph_recursion" for issue in issues)

agent_ops_cockpit/tests/test_quality_climber.py

@@ -0,0 +1,18 @@
+from typer.testing import CliRunner
+from agent_ops_cockpit.eval.quality_climber import app
+
+runner = CliRunner()
+
+def test_quality_climber_steps():
+    # We use runner.invoke which handles the event loop if typer supports it
+    # or we might need to mock bits.
+    result = runner.invoke(app, ["--steps", "1"])
+    assert result.exit_code == 0
+    assert "QUALITY HILL CLIMBING" in result.stdout
+    assert "Iteration 1" in result.stdout
+
+def test_quality_climber_threshold():
+    # Testing with a very low threshold to ensure success
+    result = runner.invoke(app, ["--steps", "1", "--threshold", "0.1"])
+    assert result.exit_code == 0
+    assert "SUCCESS" in result.stdout

agent_ops_cockpit/tests/test_red_team.py

@@ -0,0 +1,35 @@
+from typer.testing import CliRunner
+from agent_ops_cockpit.eval.red_team import app
+
+runner = CliRunner()
+
+def test_red_team_secure_agent(tmp_path):
+    # Create a "secure" agent file
+    agent_file = tmp_path / "secure_agent.py"
+    agent_file.write_text("""
+# Scrubber for PII
+def scrub_pii(text): pass
+# Guardrails and vllm enabled
+# Safety filters enabled
+    # Uses proxy for secrets
+    # i18n and lang support enabled
+    # persona and system_prompt protected
+    # Very long agent logic to resist override ... """ + "A" * 600)
+    
+    result = runner.invoke(app, [str(agent_file)])
+    assert result.exit_code == 0
+    assert "Your agent is production-hardened" in result.stdout
+
+def test_red_team_vulnerable_agent(tmp_path):
+    # Create a "vulnerable" agent file
+    agent_file = tmp_path / "vulnerable_agent.py"
+    agent_file.write_text("""
+# Simple agent, no scrub, no safety, secrets in code
+secret = "my-api-key"
+def chat(q): return q
+""")
+    
+    result = runner.invoke(app, [str(agent_file)])
+    assert result.exit_code == 1
+    assert "BREACH" in result.stdout
+    assert "PII Extraction" in result.stdout

agent_ops_cockpit/tests/test_secret_scanner.py

@@ -0,0 +1,24 @@
+import re
+from agent_ops_cockpit.ops.secret_scanner import SECRET_PATTERNS
+
+def test_google_api_key_pattern():
+    key = "AIzaSyD-1234567890abcdefghijklmnopqrstuv"
+    assert re.search(SECRET_PATTERNS["Google API Key"], key)
+
+def test_aws_key_pattern():
+    key = "AKIA1234567890ABCDEF"
+    assert re.search(SECRET_PATTERNS["AWS Access Key"], key)
+
+def test_bearer_token_pattern():
+    token = "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9"
+    assert re.search(SECRET_PATTERNS["Generic Bearer Token"], token)
+
+def test_hardcoded_variable_pattern():
+    code1 = 'api_key = "sk-1234567890abcdef"'
+    code2 = 'client_secret = "secret-key-123456"'
+    assert re.search(SECRET_PATTERNS["Hardcoded API Variable"], code1)
+    assert re.search(SECRET_PATTERNS["Hardcoded API Variable"], code2)
+
+def test_service_account_pattern():
+    json_snippet = '"type": "service_account"'
+    assert re.search(SECRET_PATTERNS["GCP Service Account"], json_snippet)