agentops-accelerator 0.3.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (142) hide show
  1. agentops/__init__.py +10 -0
  2. agentops/__main__.py +6 -0
  3. agentops/agent/__init__.py +12 -0
  4. agentops/agent/_legacy_ids.py +92 -0
  5. agentops/agent/analyzer.py +207 -0
  6. agentops/agent/checks/__init__.py +1 -0
  7. agentops/agent/checks/catalog.py +880 -0
  8. agentops/agent/checks/errors.py +279 -0
  9. agentops/agent/checks/foundry_config.py +75 -0
  10. agentops/agent/checks/latency.py +84 -0
  11. agentops/agent/checks/opex.py +157 -0
  12. agentops/agent/checks/opex_workspace.py +874 -0
  13. agentops/agent/checks/posture.py +36 -0
  14. agentops/agent/checks/posture_rules/__init__.py +53 -0
  15. agentops/agent/checks/posture_rules/content_filter.py +59 -0
  16. agentops/agent/checks/posture_rules/diagnostics.py +74 -0
  17. agentops/agent/checks/posture_rules/local_auth.py +55 -0
  18. agentops/agent/checks/posture_rules/managed_identity.py +59 -0
  19. agentops/agent/checks/posture_rules/network.py +68 -0
  20. agentops/agent/checks/regression.py +78 -0
  21. agentops/agent/checks/release_readiness.py +182 -0
  22. agentops/agent/checks/safety.py +247 -0
  23. agentops/agent/checks/spec_conformance.py +375 -0
  24. agentops/agent/cockpit.py +5159 -0
  25. agentops/agent/config.py +240 -0
  26. agentops/agent/findings.py +113 -0
  27. agentops/agent/history.py +142 -0
  28. agentops/agent/knowledge/__init__.py +182 -0
  29. agentops/agent/knowledge/waf-checklist.csv +39 -0
  30. agentops/agent/llm_assist/__init__.py +16 -0
  31. agentops/agent/llm_assist/_base.py +124 -0
  32. agentops/agent/llm_assist/_bundle_rule.py +154 -0
  33. agentops/agent/llm_assist/_client.py +347 -0
  34. agentops/agent/llm_assist/_dataset_rules.py +191 -0
  35. agentops/agent/llm_assist/_engine.py +106 -0
  36. agentops/agent/llm_assist/_prompt_rules.py +291 -0
  37. agentops/agent/llm_assist/_spec_rules.py +235 -0
  38. agentops/agent/production_telemetry.py +430 -0
  39. agentops/agent/report.py +207 -0
  40. agentops/agent/server/__init__.py +1 -0
  41. agentops/agent/server/app.py +84 -0
  42. agentops/agent/server/auth.py +94 -0
  43. agentops/agent/server/chat.py +44 -0
  44. agentops/agent/server/protocol.py +72 -0
  45. agentops/agent/sources/__init__.py +1 -0
  46. agentops/agent/sources/azure_monitor.py +523 -0
  47. agentops/agent/sources/azure_resources.py +602 -0
  48. agentops/agent/sources/foundry_control.py +174 -0
  49. agentops/agent/sources/results_history.py +494 -0
  50. agentops/agent/sources/spec_detectors/__init__.py +42 -0
  51. agentops/agent/sources/spec_detectors/_base.py +58 -0
  52. agentops/agent/sources/spec_detectors/agents_md.py +75 -0
  53. agentops/agent/sources/spec_detectors/spec_kit.py +172 -0
  54. agentops/agent/time_range.py +117 -0
  55. agentops/cli/__init__.py +1 -0
  56. agentops/cli/app.py +4823 -0
  57. agentops/core/__init__.py +1 -0
  58. agentops/core/agentops_config.py +592 -0
  59. agentops/core/config_loader.py +22 -0
  60. agentops/core/evaluators.py +480 -0
  61. agentops/core/release_evidence.py +56 -0
  62. agentops/core/results.py +117 -0
  63. agentops/mcp/__init__.py +10 -0
  64. agentops/mcp/server.py +232 -0
  65. agentops/pipeline/__init__.py +8 -0
  66. agentops/pipeline/cloud_results.py +189 -0
  67. agentops/pipeline/cloud_runner.py +901 -0
  68. agentops/pipeline/comparison.py +108 -0
  69. agentops/pipeline/diagnostics.py +51 -0
  70. agentops/pipeline/invocations.py +535 -0
  71. agentops/pipeline/official_eval.py +414 -0
  72. agentops/pipeline/orchestrator.py +775 -0
  73. agentops/pipeline/prompt_deploy.py +377 -0
  74. agentops/pipeline/publisher.py +121 -0
  75. agentops/pipeline/reporter.py +202 -0
  76. agentops/pipeline/runtime.py +409 -0
  77. agentops/pipeline/thresholds.py +84 -0
  78. agentops/services/__init__.py +1 -0
  79. agentops/services/cicd.py +720 -0
  80. agentops/services/eval_analysis.py +848 -0
  81. agentops/services/evidence_pack.py +757 -0
  82. agentops/services/initializer.py +86 -0
  83. agentops/services/preflight.py +470 -0
  84. agentops/services/setup_wizard.py +709 -0
  85. agentops/services/skills.py +643 -0
  86. agentops/services/trace_promotion.py +300 -0
  87. agentops/services/workflow_analysis.py +1129 -0
  88. agentops/templates/.gitignore +15 -0
  89. agentops/templates/__init__.py +1 -0
  90. agentops/templates/agent-server/Dockerfile +23 -0
  91. agentops/templates/agent-server/README.md +61 -0
  92. agentops/templates/agent-server/main.bicep +94 -0
  93. agentops/templates/agent.yaml +87 -0
  94. agentops/templates/agentops.yaml +58 -0
  95. agentops/templates/foundry.svg +71 -0
  96. agentops/templates/icon.png +0 -0
  97. agentops/templates/pipelines/azuredevops/agentops-deploy-dev-azd.yml +118 -0
  98. agentops/templates/pipelines/azuredevops/agentops-deploy-dev.yml +73 -0
  99. agentops/templates/pipelines/azuredevops/agentops-deploy-prod-azd.yml +141 -0
  100. agentops/templates/pipelines/azuredevops/agentops-deploy-prod.yml +94 -0
  101. agentops/templates/pipelines/azuredevops/agentops-deploy-prompt-agent.yml +167 -0
  102. agentops/templates/pipelines/azuredevops/agentops-deploy-qa-azd.yml +118 -0
  103. agentops/templates/pipelines/azuredevops/agentops-deploy-qa.yml +68 -0
  104. agentops/templates/pipelines/azuredevops/agentops-pr-prompt-agent.yml +210 -0
  105. agentops/templates/pipelines/azuredevops/agentops-pr.yml +155 -0
  106. agentops/templates/pipelines/azuredevops/agentops-watchdog.yml +106 -0
  107. agentops/templates/project.gitignore +36 -0
  108. agentops/templates/sample-traces.jsonl +3 -0
  109. agentops/templates/skills/agentops-agent/SKILL.md +137 -0
  110. agentops/templates/skills/agentops-config/SKILL.md +113 -0
  111. agentops/templates/skills/agentops-dataset/SKILL.md +84 -0
  112. agentops/templates/skills/agentops-eval/SKILL.md +189 -0
  113. agentops/templates/skills/agentops-report/SKILL.md +71 -0
  114. agentops/templates/skills/agentops-workflow/SKILL.md +471 -0
  115. agentops/templates/smoke.jsonl +3 -0
  116. agentops/templates/waf-checklist.README.md +84 -0
  117. agentops/templates/waf-checklist.csv +22 -0
  118. agentops/templates/workflows/agentops-deploy-dev-azd.yml +166 -0
  119. agentops/templates/workflows/agentops-deploy-dev.yml +187 -0
  120. agentops/templates/workflows/agentops-deploy-prod-azd.yml +183 -0
  121. agentops/templates/workflows/agentops-deploy-prod.yml +171 -0
  122. agentops/templates/workflows/agentops-deploy-prompt-agent.yml +197 -0
  123. agentops/templates/workflows/agentops-deploy-qa-azd.yml +156 -0
  124. agentops/templates/workflows/agentops-deploy-qa.yml +145 -0
  125. agentops/templates/workflows/agentops-pr-prompt-agent.yml +210 -0
  126. agentops/templates/workflows/agentops-pr.yml +148 -0
  127. agentops/templates/workflows/agentops-watchdog.yml +122 -0
  128. agentops/utils/__init__.py +1 -0
  129. agentops/utils/azd_env.py +435 -0
  130. agentops/utils/azure_endpoints.py +62 -0
  131. agentops/utils/colors.py +47 -0
  132. agentops/utils/dotenv_loader.py +105 -0
  133. agentops/utils/foundry_discovery.py +229 -0
  134. agentops/utils/logging.py +59 -0
  135. agentops/utils/telemetry.py +554 -0
  136. agentops/utils/yaml.py +36 -0
  137. agentops_accelerator-0.3.0.dist-info/METADATA +278 -0
  138. agentops_accelerator-0.3.0.dist-info/RECORD +142 -0
  139. agentops_accelerator-0.3.0.dist-info/WHEEL +5 -0
  140. agentops_accelerator-0.3.0.dist-info/entry_points.txt +2 -0
  141. agentops_accelerator-0.3.0.dist-info/licenses/LICENSE +21 -0
  142. agentops_accelerator-0.3.0.dist-info/top_level.txt +1 -0
agentops/agent/server/app.py ADDED
@@ -0,0 +1,84 @@
1
+ """FastAPI factory for the Copilot Extension server.
2
+
3
+ Exposes:
4
+ - ``POST /agents/messages`` - Copilot Extensions protocol (SSE response)
5
+ - ``GET /healthz`` - liveness check
6
+ - ``GET /`` - small HTML index
7
+ """
8
+
9
+ import logging
10
+ from pathlib import Path
11
+
12
+ from agentops.agent.config import AgentConfig
13
+
14
+ log = logging.getLogger(__name__)
15
+
16
+
17
+ def create_app(
18
+ workspace: Path,
19
+ config: AgentConfig,
20
+ verify_signature: bool = True,
21
+ ):
22
+ """Build a FastAPI app for the watchdog Copilot Extension server."""
23
+ try:
24
+ from fastapi import FastAPI, HTTPException, Request
25
+ from fastapi.responses import HTMLResponse, JSONResponse, StreamingResponse
26
+ except ImportError as exc: # pragma: no cover - import guard
27
+ raise RuntimeError(
28
+ "FastAPI is not installed. Install agent extras with "
29
+ "'pip install agentops-accelerator[agent]'."
30
+ ) from exc
31
+
32
+ from agentops.agent.server.chat import stream_reply
33
+ from agentops.agent.server.protocol import parse_copilot_request
34
+
35
+ app = FastAPI(title="AgentOps Doctor", version="1.0")
36
+
37
+ @app.get("/healthz")
38
+ def healthz() -> JSONResponse:
39
+ return JSONResponse({"status": "ok"})
40
+
41
+ @app.get("/", response_class=HTMLResponse)
42
+ def index() -> str:
43
+ return (
44
+ "<!doctype html><html><body>"
45
+ "<h1>AgentOps Doctor</h1>"
46
+ "<p>Copilot Extension endpoint: <code>POST /agents/messages</code></p>"
47
+ "<p>Health: <a href='/healthz'>/healthz</a></p>"
48
+ "</body></html>"
49
+ )
50
+
51
+ @app.post("/agents/messages")
52
+ async def messages(request: Request):
53
+ body_bytes = await request.body()
54
+
55
+ if verify_signature:
56
+ from agentops.agent.server.auth import verify_signature as _verify
57
+
58
+ try:
59
+ _verify(
60
+ body_bytes,
61
+ request.headers.get("x-github-public-key-identifier"),
62
+ request.headers.get("x-github-public-key-signature"),
63
+ )
64
+ except ValueError as exc:
65
+ raise HTTPException(status_code=401, detail=str(exc)) from exc
66
+
67
+ try:
68
+ payload = await request.json()
69
+ except Exception as exc:
70
+ raise HTTPException(
71
+ status_code=400, detail=f"invalid JSON body: {exc}"
72
+ ) from exc
73
+
74
+ copilot_request = parse_copilot_request(
75
+ payload if isinstance(payload, dict) else {}
76
+ )
77
+
78
+ def _generator():
79
+ for chunk in stream_reply(workspace, config, copilot_request):
80
+ yield chunk
81
+
82
+ return StreamingResponse(_generator(), media_type="text/event-stream")
83
+
84
+ return app
agentops/agent/server/auth.py ADDED
@@ -0,0 +1,94 @@
1
+ """Copilot Extension request signature validation.
2
+
3
+ Validates the ``X-GitHub-Public-Key-Identifier`` and
4
+ ``X-GitHub-Public-Key-Signature`` headers against GitHub's published
5
+ public keys. The validation can be disabled (``--no-verify``) for local
6
+ development and tests.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import logging
12
+ import time
13
+ from dataclasses import dataclass, field
14
+ from typing import Dict, Optional
15
+
16
+ log = logging.getLogger(__name__)
17
+
18
+ GITHUB_KEYS_URL = "https://api.github.com/meta/public_keys/copilot_api"
19
+ KEY_CACHE_TTL_SECONDS = 60 * 30 # 30 minutes
20
+
21
+
22
+ @dataclass
23
+ class _KeyCache:
24
+ keys: Dict[str, str] = field(default_factory=dict)
25
+ fetched_at: float = 0.0
26
+
27
+
28
+ _cache = _KeyCache()
29
+
30
+
31
+ def _fetch_keys() -> Dict[str, str]:
32
+ import httpx # local import keeps base CLI lean
33
+
34
+ with httpx.Client(timeout=10.0) as client:
35
+ response = client.get(GITHUB_KEYS_URL)
36
+ response.raise_for_status()
37
+ payload = response.json()
38
+ keys = {}
39
+ for entry in payload.get("public_keys", []):
40
+ identifier = entry.get("key_identifier")
41
+ key = entry.get("key")
42
+ if identifier and key:
43
+ keys[identifier] = key
44
+ return keys
45
+
46
+
47
+ def _get_keys(force_refresh: bool = False) -> Dict[str, str]:
48
+ now = time.time()
49
+ if (
50
+ force_refresh
51
+ or not _cache.keys
52
+ or now - _cache.fetched_at > KEY_CACHE_TTL_SECONDS
53
+ ):
54
+ _cache.keys = _fetch_keys()
55
+ _cache.fetched_at = now
56
+ return _cache.keys
57
+
58
+
59
+ def verify_signature(
60
+ body: bytes,
61
+ key_identifier: Optional[str],
62
+ signature_b64: Optional[str],
63
+ ) -> None:
64
+ """Raise ``ValueError`` if the request signature is invalid."""
65
+ if not key_identifier or not signature_b64:
66
+ raise ValueError("missing Copilot signature headers")
67
+
68
+ import base64
69
+
70
+ from cryptography.exceptions import InvalidSignature
71
+ from cryptography.hazmat.primitives import hashes, serialization
72
+ from cryptography.hazmat.primitives.asymmetric import ec
73
+
74
+ keys = _get_keys()
75
+ pem = keys.get(key_identifier)
76
+ if pem is None:
77
+ keys = _get_keys(force_refresh=True)
78
+ pem = keys.get(key_identifier)
79
+ if pem is None:
80
+ raise ValueError(f"unknown key identifier {key_identifier!r}")
81
+
82
+ public_key = serialization.load_pem_public_key(pem.encode("utf-8"))
83
+ if not isinstance(public_key, ec.EllipticCurvePublicKey):
84
+ raise ValueError("Copilot public key is not an EC key")
85
+
86
+ try:
87
+ signature = base64.b64decode(signature_b64)
88
+ except Exception as exc: # pragma: no cover - malformed inputs
89
+ raise ValueError(f"invalid signature encoding: {exc}") from exc
90
+
91
+ try:
92
+ public_key.verify(signature, body, ec.ECDSA(hashes.SHA256()))
93
+ except InvalidSignature as exc:
94
+ raise ValueError("signature verification failed") from exc
agentops/agent/server/chat.py ADDED
@@ -0,0 +1,44 @@
1
+ """Chat orchestration: turns a Copilot user message into an SSE reply."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from pathlib import Path
6
+ from typing import Iterable
7
+
8
+ from agentops.agent.analyzer import analyze
9
+ from agentops.agent.config import AgentConfig
10
+ from agentops.agent.report import render_report, short_chat_summary
11
+ from agentops.agent.server.protocol import CopilotRequest, stream_markdown
12
+
13
+
14
+ def _intro_for(message: str) -> str:
15
+ msg = (message or "").lower()
16
+ if any(word in msg for word in ("regress", "drop", "score")):
17
+ focus = "regressions"
18
+ elif any(word in msg for word in ("latency", "slow", "p95")):
19
+ focus = "latency"
20
+ elif any(word in msg for word in ("error", "fail", "exception")):
21
+ focus = "production errors"
22
+ elif any(word in msg for word in ("safety", "harm", "violen")):
23
+ focus = "content safety"
24
+ else:
25
+ focus = "agent health"
26
+ return (
27
+ f"I scanned your AgentOps eval history, Azure Monitor, and Foundry "
28
+ f"control plane focused on **{focus}**.\n\n"
29
+ )
30
+
31
+
32
+ def build_reply(workspace: Path, config: AgentConfig, request: CopilotRequest) -> str:
33
+ user_message = request.last_user_message or ""
34
+ result = analyze(workspace, config)
35
+ intro = _intro_for(user_message)
36
+ summary = short_chat_summary(result)
37
+ report = render_report(result)
38
+ return f"{intro}{summary}\n\n---\n\n{report}"
39
+
40
+
41
+ def stream_reply(
42
+ workspace: Path, config: AgentConfig, request: CopilotRequest
43
+ ) -> Iterable[bytes]:
44
+ return stream_markdown(build_reply(workspace, config, request))
agentops/agent/server/protocol.py ADDED
@@ -0,0 +1,72 @@
1
+ """GitHub Copilot Extensions request/response protocol helpers."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from dataclasses import dataclass
6
+ from typing import Any, Dict, Iterable, List, Optional
7
+
8
+ import json
9
+
10
+
11
+ @dataclass
12
+ class CopilotMessage:
13
+ role: str
14
+ content: str
15
+
16
+
17
+ @dataclass
18
+ class CopilotRequest:
19
+ messages: List[CopilotMessage]
20
+ raw: Dict[str, Any]
21
+
22
+ @property
23
+ def last_user_message(self) -> Optional[str]:
24
+ for message in reversed(self.messages):
25
+ if message.role == "user" and message.content:
26
+ return message.content
27
+ return None
28
+
29
+
30
+ def parse_copilot_request(body: Dict[str, Any]) -> CopilotRequest:
31
+ raw_messages = body.get("messages") or []
32
+ messages: List[CopilotMessage] = []
33
+ if isinstance(raw_messages, list):
34
+ for entry in raw_messages:
35
+ if not isinstance(entry, dict):
36
+ continue
37
+ role = str(entry.get("role") or "user")
38
+ content = entry.get("content") or ""
39
+ if isinstance(content, list):
40
+ # Multipart content -> concatenate text parts.
41
+ parts: List[str] = []
42
+ for part in content:
43
+ if isinstance(part, dict) and part.get("type") == "text":
44
+ parts.append(str(part.get("text", "")))
45
+ content = "".join(parts)
46
+ messages.append(CopilotMessage(role=role, content=str(content)))
47
+ return CopilotRequest(messages=messages, raw=body)
48
+
49
+
50
+ def sse_text_chunk(text: str) -> bytes:
51
+ payload = {
52
+ "choices": [
53
+ {
54
+ "delta": {"role": "assistant", "content": text},
55
+ "index": 0,
56
+ }
57
+ ]
58
+ }
59
+ return f"data: {json.dumps(payload)}\n\n".encode("utf-8")
60
+
61
+
62
+ def sse_done() -> bytes:
63
+ return b"data: [DONE]\n\n"
64
+
65
+
66
+ def stream_markdown(markdown: str, chunk_size: int = 512) -> Iterable[bytes]:
67
+ """Yield SSE chunks for a Markdown reply, then a [DONE] sentinel."""
68
+ if not markdown:
69
+ markdown = "_(empty reply)_"
70
+ for start in range(0, len(markdown), chunk_size):
71
+ yield sse_text_chunk(markdown[start : start + chunk_size])
72
+ yield sse_done()
agentops/agent/sources/__init__.py ADDED
@@ -0,0 +1 @@
1
+ """Watchdog agent signal sources."""