agentra 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agentra/__init__.py +3 -0
- agentra/adapters/__init__.py +0 -0
- agentra/adapters/agents.py +231 -0
- agentra/benchmarks/__init__.py +0 -0
- agentra/benchmarks/runner.py +243 -0
- agentra/cli/__init__.py +0 -0
- agentra/cli/main.py +418 -0
- agentra/compliance/__init__.py +0 -0
- agentra/compliance/engine.py +82 -0
- agentra/detection/__init__.py +0 -0
- agentra/detection/engine.py +189 -0
- agentra/execution/__init__.py +0 -0
- agentra/execution/engine.py +187 -0
- agentra/governance/__init__.py +0 -0
- agentra/governance/engine.py +130 -0
- agentra/governance/policies.py +326 -0
- agentra/models.py +237 -0
- agentra/onboarding/__init__.py +0 -0
- agentra/onboarding/engine.py +134 -0
- agentra/optimizer/__init__.py +0 -0
- agentra/optimizer/engine.py +136 -0
- agentra/renderers/__init__.py +0 -0
- agentra/renderers/html.py +209 -0
- agentra/renderers/markdown.py +125 -0
- agentra/risk/__init__.py +0 -0
- agentra/risk/engine.py +47 -0
- agentra/skills/__init__.py +0 -0
- agentra/skills/registry.py +351 -0
- agentra/telemetry/__init__.py +0 -0
- agentra/telemetry/audit.py +41 -0
- agentra-0.1.0.dist-info/METADATA +267 -0
- agentra-0.1.0.dist-info/RECORD +36 -0
- agentra-0.1.0.dist-info/WHEEL +5 -0
- agentra-0.1.0.dist-info/entry_points.txt +2 -0
- agentra-0.1.0.dist-info/licenses/LICENSE +21 -0
- agentra-0.1.0.dist-info/top_level.txt +1 -0
agentra/__init__.py
ADDED
|
File without changes
|
|
@@ -0,0 +1,231 @@
|
|
|
1
|
+
"""Agent Integration Adapters — generate optimized configs for each agent platform."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import Protocol
|
|
7
|
+
|
|
8
|
+
from agentra.models import AgentPlatform, ProjectConfig, StackProfile
|
|
9
|
+
from agentra.governance.engine import GovernanceEngine
|
|
10
|
+
from agentra.optimizer.engine import TokenOptimizer
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class AgentAdapter(Protocol):
|
|
14
|
+
"""Protocol for agent-specific output adapters."""
|
|
15
|
+
|
|
16
|
+
platform: AgentPlatform
|
|
17
|
+
|
|
18
|
+
def generate(
|
|
19
|
+
self,
|
|
20
|
+
config: ProjectConfig,
|
|
21
|
+
stack: StackProfile,
|
|
22
|
+
governance: GovernanceEngine,
|
|
23
|
+
optimizer: TokenOptimizer,
|
|
24
|
+
) -> dict[str, str]:
|
|
25
|
+
"""Return {filename: content} for this agent platform."""
|
|
26
|
+
...
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
# ── Shared helpers ───────────────────────────────────────────────────────────
|
|
30
|
+
|
|
31
|
+
def _build_header(platform_name: str) -> str:
|
|
32
|
+
return (
|
|
33
|
+
f"# Agentra — {platform_name} Instructions\n"
|
|
34
|
+
f"# Auto-generated. Do not edit manually.\n"
|
|
35
|
+
f"# Regenerate with: ag init\n\n"
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _build_security_block(governance: GovernanceEngine, optimizer: TokenOptimizer) -> str:
|
|
40
|
+
instructions = governance.generate_instructions()
|
|
41
|
+
compressed = optimizer.compress_instructions(instructions)
|
|
42
|
+
return f"## Security & Governance\n{compressed}\n"
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _build_stack_block(stack: StackProfile) -> str:
|
|
46
|
+
lines = ["## Detected Stack"]
|
|
47
|
+
for cat, label in [
|
|
48
|
+
("languages", "Languages"),
|
|
49
|
+
("frameworks", "Frameworks"),
|
|
50
|
+
("databases", "Databases"),
|
|
51
|
+
("sdks", "SDKs"),
|
|
52
|
+
("infrastructure", "Infrastructure"),
|
|
53
|
+
]:
|
|
54
|
+
components = getattr(stack, cat)
|
|
55
|
+
if components:
|
|
56
|
+
names = ", ".join(c.name for c in components)
|
|
57
|
+
lines.append(f"- **{label}**: {names}")
|
|
58
|
+
return "\n".join(lines) + "\n"
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _build_skills_block(config: ProjectConfig) -> str:
|
|
62
|
+
if not config.skills:
|
|
63
|
+
return ""
|
|
64
|
+
lines = ["## Active Skills"]
|
|
65
|
+
for s in config.skills:
|
|
66
|
+
lines.append(f"- {s}")
|
|
67
|
+
return "\n".join(lines) + "\n"
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
# ── Claude Adapter ───────────────────────────────────────────────────────────
|
|
71
|
+
|
|
72
|
+
class ClaudeAdapter:
|
|
73
|
+
platform = AgentPlatform.CLAUDE
|
|
74
|
+
|
|
75
|
+
def generate(self, config: ProjectConfig, stack: StackProfile,
|
|
76
|
+
governance: GovernanceEngine, optimizer: TokenOptimizer) -> dict[str, str]:
|
|
77
|
+
parts = [
|
|
78
|
+
_build_header("Claude Code (CLAUDE.md)"),
|
|
79
|
+
_build_stack_block(stack),
|
|
80
|
+
_build_security_block(governance, optimizer),
|
|
81
|
+
_build_skills_block(config),
|
|
82
|
+
]
|
|
83
|
+
return {"CLAUDE.md": "\n".join(parts)}
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
# ── Cursor Adapter ───────────────────────────────────────────────────────────
|
|
87
|
+
|
|
88
|
+
class CursorAdapter:
|
|
89
|
+
platform = AgentPlatform.CURSOR
|
|
90
|
+
|
|
91
|
+
def generate(self, config: ProjectConfig, stack: StackProfile,
|
|
92
|
+
governance: GovernanceEngine, optimizer: TokenOptimizer) -> dict[str, str]:
|
|
93
|
+
parts = [
|
|
94
|
+
_build_header("Cursor (.cursorrules)"),
|
|
95
|
+
_build_stack_block(stack),
|
|
96
|
+
_build_security_block(governance, optimizer),
|
|
97
|
+
_build_skills_block(config),
|
|
98
|
+
]
|
|
99
|
+
return {".cursorrules": "\n".join(parts)}
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
# ── GitHub Copilot Adapter ───────────────────────────────────────────────────
|
|
103
|
+
|
|
104
|
+
class CopilotAdapter:
|
|
105
|
+
platform = AgentPlatform.COPILOT
|
|
106
|
+
|
|
107
|
+
def generate(self, config: ProjectConfig, stack: StackProfile,
|
|
108
|
+
governance: GovernanceEngine, optimizer: TokenOptimizer) -> dict[str, str]:
|
|
109
|
+
parts = [
|
|
110
|
+
_build_header("GitHub Copilot"),
|
|
111
|
+
_build_stack_block(stack),
|
|
112
|
+
_build_security_block(governance, optimizer),
|
|
113
|
+
_build_skills_block(config),
|
|
114
|
+
]
|
|
115
|
+
return {".github/copilot-instructions.md": "\n".join(parts)}
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
# ── Aider Adapter ────────────────────────────────────────────────────────────
|
|
119
|
+
|
|
120
|
+
class AiderAdapter:
|
|
121
|
+
platform = AgentPlatform.AIDER
|
|
122
|
+
|
|
123
|
+
def generate(self, config: ProjectConfig, stack: StackProfile,
|
|
124
|
+
governance: GovernanceEngine, optimizer: TokenOptimizer) -> dict[str, str]:
|
|
125
|
+
parts = [
|
|
126
|
+
_build_header("Aider (.aider.conf.yml)"),
|
|
127
|
+
_build_stack_block(stack),
|
|
128
|
+
_build_security_block(governance, optimizer),
|
|
129
|
+
]
|
|
130
|
+
content = "\n".join(parts)
|
|
131
|
+
# Wrap in YAML conventions block
|
|
132
|
+
yaml_content = f"# Aider conventions\nconventions: |\n"
|
|
133
|
+
for line in content.splitlines():
|
|
134
|
+
yaml_content += f" {line}\n"
|
|
135
|
+
return {".aider.conf.yml": yaml_content}
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
# ── Windsurf Adapter ─────────────────────────────────────────────────────────
|
|
139
|
+
|
|
140
|
+
class WindsurfAdapter:
|
|
141
|
+
platform = AgentPlatform.WINDSURF
|
|
142
|
+
|
|
143
|
+
def generate(self, config: ProjectConfig, stack: StackProfile,
|
|
144
|
+
governance: GovernanceEngine, optimizer: TokenOptimizer) -> dict[str, str]:
|
|
145
|
+
parts = [
|
|
146
|
+
_build_header("Windsurf"),
|
|
147
|
+
_build_stack_block(stack),
|
|
148
|
+
_build_security_block(governance, optimizer),
|
|
149
|
+
_build_skills_block(config),
|
|
150
|
+
]
|
|
151
|
+
return {".windsurfrules": "\n".join(parts)}
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
# ── Continue.dev Adapter ─────────────────────────────────────────────────────
|
|
155
|
+
|
|
156
|
+
class ContinueAdapter:
|
|
157
|
+
platform = AgentPlatform.CONTINUE
|
|
158
|
+
|
|
159
|
+
def generate(self, config: ProjectConfig, stack: StackProfile,
|
|
160
|
+
governance: GovernanceEngine, optimizer: TokenOptimizer) -> dict[str, str]:
|
|
161
|
+
import json
|
|
162
|
+
instructions = governance.generate_instructions()
|
|
163
|
+
compressed = optimizer.compress_instructions(instructions)
|
|
164
|
+
cfg = {
|
|
165
|
+
"systemMessage": compressed[:4000],
|
|
166
|
+
"models": [],
|
|
167
|
+
}
|
|
168
|
+
return {".continue/config.json": json.dumps(cfg, indent=2)}
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
# ── AGENTS.md Adapter (universal) ────────────────────────────────────────────
|
|
172
|
+
|
|
173
|
+
class AgentsMdAdapter:
|
|
174
|
+
platform = AgentPlatform.CLAUDE # generic
|
|
175
|
+
|
|
176
|
+
def generate(self, config: ProjectConfig, stack: StackProfile,
|
|
177
|
+
governance: GovernanceEngine, optimizer: TokenOptimizer) -> dict[str, str]:
|
|
178
|
+
parts = [
|
|
179
|
+
_build_header("AGENTS.md — Universal Agent Instructions"),
|
|
180
|
+
_build_stack_block(stack),
|
|
181
|
+
_build_security_block(governance, optimizer),
|
|
182
|
+
_build_skills_block(config),
|
|
183
|
+
"\n## Execution Safety\n"
|
|
184
|
+
"- Always dry-run destructive commands first\n"
|
|
185
|
+
"- Never execute code that modifies production data without approval\n"
|
|
186
|
+
"- Sandbox all generated code execution\n"
|
|
187
|
+
"- Create rollback scripts before schema changes\n",
|
|
188
|
+
]
|
|
189
|
+
return {"AGENTS.md": "\n".join(parts)}
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
# ── Registry ─────────────────────────────────────────────────────────────────
|
|
193
|
+
|
|
194
|
+
ADAPTER_REGISTRY: dict[AgentPlatform, AgentAdapter] = {
|
|
195
|
+
AgentPlatform.CLAUDE: ClaudeAdapter(),
|
|
196
|
+
AgentPlatform.CURSOR: CursorAdapter(),
|
|
197
|
+
AgentPlatform.COPILOT: CopilotAdapter(),
|
|
198
|
+
AgentPlatform.AIDER: AiderAdapter(),
|
|
199
|
+
AgentPlatform.WINDSURF: WindsurfAdapter(),
|
|
200
|
+
AgentPlatform.CONTINUE: ContinueAdapter(),
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def generate_for_agents(
|
|
205
|
+
agents: list[AgentPlatform],
|
|
206
|
+
config: ProjectConfig,
|
|
207
|
+
stack: StackProfile,
|
|
208
|
+
governance: GovernanceEngine,
|
|
209
|
+
optimizer: TokenOptimizer,
|
|
210
|
+
) -> dict[str, str]:
|
|
211
|
+
"""Generate config files for all requested agents. Returns {path: content}."""
|
|
212
|
+
outputs: dict[str, str] = {}
|
|
213
|
+
for agent in agents:
|
|
214
|
+
adapter = ADAPTER_REGISTRY.get(agent)
|
|
215
|
+
if adapter:
|
|
216
|
+
outputs.update(adapter.generate(config, stack, governance, optimizer))
|
|
217
|
+
# Always include AGENTS.md
|
|
218
|
+
agents_adapter = AgentsMdAdapter()
|
|
219
|
+
outputs.update(agents_adapter.generate(config, stack, governance, optimizer))
|
|
220
|
+
return outputs
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
def write_agent_files(output_dir: Path, files: dict[str, str]) -> list[Path]:
|
|
224
|
+
"""Write generated agent files to disk."""
|
|
225
|
+
written: list[Path] = []
|
|
226
|
+
for rel_path, content in files.items():
|
|
227
|
+
fp = output_dir / rel_path
|
|
228
|
+
fp.parent.mkdir(parents=True, exist_ok=True)
|
|
229
|
+
fp.write_text(content, encoding="utf-8")
|
|
230
|
+
written.append(fp)
|
|
231
|
+
return written
|
|
File without changes
|
|
@@ -0,0 +1,243 @@
|
|
|
1
|
+
"""Benchmark Runner — measures before/after metrics for each skill."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import time
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
from agentra.models import (
|
|
9
|
+
BenchmarkMetric,
|
|
10
|
+
BenchmarkReport,
|
|
11
|
+
OptimizationResult,
|
|
12
|
+
SkillBenchmark,
|
|
13
|
+
)
|
|
14
|
+
from agentra.detection.engine import StackDetector
|
|
15
|
+
from agentra.governance.engine import GovernanceEngine
|
|
16
|
+
from agentra.governance.policies import ALL_POLICIES, get_policies_for_stack
|
|
17
|
+
from agentra.optimizer.engine import TokenOptimizer
|
|
18
|
+
from agentra.skills.registry import BUILTIN_SKILLS, SkillRegistry
|
|
19
|
+
from agentra.compliance.engine import ComplianceEngine
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class BenchmarkRunner:
|
|
23
|
+
"""Runs before/after benchmarks for each skill to quantify improvement."""
|
|
24
|
+
|
|
25
|
+
def __init__(self, project_root: Path | str):
|
|
26
|
+
self.root = Path(project_root).resolve()
|
|
27
|
+
self.registry = SkillRegistry()
|
|
28
|
+
|
|
29
|
+
def run(self) -> BenchmarkReport:
|
|
30
|
+
"""Execute full benchmark suite."""
|
|
31
|
+
detector = StackDetector(self.root)
|
|
32
|
+
stack = detector.detect()
|
|
33
|
+
stack_names = [c.name for c in stack.all_components] or ["all"]
|
|
34
|
+
|
|
35
|
+
# Baseline governance (no skills applied)
|
|
36
|
+
gov_engine = GovernanceEngine(stack)
|
|
37
|
+
baseline_result = gov_engine.enforce(self.root)
|
|
38
|
+
|
|
39
|
+
# Baseline optimization
|
|
40
|
+
all_policies = get_policies_for_stack(stack_names)
|
|
41
|
+
optimizer = TokenOptimizer()
|
|
42
|
+
baseline_opt = optimizer.optimize(all_policies, stack)
|
|
43
|
+
|
|
44
|
+
# Resolve applicable skills
|
|
45
|
+
applicable_skills = self.registry.resolve_for_stack(stack_names)
|
|
46
|
+
|
|
47
|
+
skill_benchmarks: list[SkillBenchmark] = []
|
|
48
|
+
for skill in applicable_skills:
|
|
49
|
+
sb = self._benchmark_skill(skill, stack_names, baseline_opt, baseline_result)
|
|
50
|
+
skill_benchmarks.append(sb)
|
|
51
|
+
|
|
52
|
+
# Also benchmark the governance engine itself
|
|
53
|
+
gov_benchmark = self._benchmark_governance(baseline_result)
|
|
54
|
+
skill_benchmarks.append(gov_benchmark)
|
|
55
|
+
|
|
56
|
+
# Benchmark the optimization engine
|
|
57
|
+
opt_benchmark = self._benchmark_optimization(baseline_opt)
|
|
58
|
+
skill_benchmarks.append(opt_benchmark)
|
|
59
|
+
|
|
60
|
+
return BenchmarkReport(
|
|
61
|
+
project_name=self.root.name,
|
|
62
|
+
stack=stack,
|
|
63
|
+
governance=baseline_result,
|
|
64
|
+
optimization=baseline_opt,
|
|
65
|
+
skill_benchmarks=skill_benchmarks,
|
|
66
|
+
)
|
|
67
|
+
|
|
68
|
+
def _benchmark_skill(
|
|
69
|
+
self, skill, stack_names: list[str],
|
|
70
|
+
baseline_opt: OptimizationResult, baseline_result
|
|
71
|
+
) -> SkillBenchmark:
|
|
72
|
+
"""Benchmark a single skill: measure token cost, coverage, relevance."""
|
|
73
|
+
from agentra.optimizer.engine import _estimate_tokens
|
|
74
|
+
|
|
75
|
+
metrics: list[BenchmarkMetric] = []
|
|
76
|
+
|
|
77
|
+
# ── 1. Token cost of skill instructions ──────────────────────────
|
|
78
|
+
skill_tokens = _estimate_tokens(skill.instructions) if skill.instructions else 0
|
|
79
|
+
# Without skill: 0 tokens for this skill, with skill: skill_tokens
|
|
80
|
+
metrics.append(BenchmarkMetric(
|
|
81
|
+
name="Instruction Token Cost",
|
|
82
|
+
before=0,
|
|
83
|
+
after=skill_tokens,
|
|
84
|
+
unit="tokens",
|
|
85
|
+
improvement_pct=0,
|
|
86
|
+
description=f"Tokens consumed by {skill.name} instructions.",
|
|
87
|
+
))
|
|
88
|
+
|
|
89
|
+
# ── 2. Policy coverage ───────────────────────────────────────────
|
|
90
|
+
all_policy_ids = {p.id for p in ALL_POLICIES}
|
|
91
|
+
skill_policy_ids = set(skill.policies) & all_policy_ids
|
|
92
|
+
before_coverage = 0
|
|
93
|
+
after_coverage = len(skill_policy_ids)
|
|
94
|
+
metrics.append(BenchmarkMetric(
|
|
95
|
+
name="Security Policy Coverage",
|
|
96
|
+
before=before_coverage,
|
|
97
|
+
after=after_coverage,
|
|
98
|
+
unit="policies",
|
|
99
|
+
improvement_pct=100.0 if after_coverage > 0 else 0,
|
|
100
|
+
description=f"Security policies activated by {skill.name}.",
|
|
101
|
+
))
|
|
102
|
+
|
|
103
|
+
# ── 3. Context relevance score ───────────────────────────────────
|
|
104
|
+
# Score based on stack match
|
|
105
|
+
stack_lower = {s.lower() for s in stack_names}
|
|
106
|
+
skill_stacks = {s.lower() for s in skill.stacks}
|
|
107
|
+
if "all" in skill_stacks:
|
|
108
|
+
relevance = 0.8
|
|
109
|
+
elif skill_stacks & stack_lower:
|
|
110
|
+
relevance = 1.0
|
|
111
|
+
else:
|
|
112
|
+
relevance = 0.2
|
|
113
|
+
|
|
114
|
+
metrics.append(BenchmarkMetric(
|
|
115
|
+
name="Context Relevance",
|
|
116
|
+
before=0.0,
|
|
117
|
+
after=round(relevance, 2),
|
|
118
|
+
unit="score (0-1)",
|
|
119
|
+
improvement_pct=round(relevance * 100, 1),
|
|
120
|
+
description=f"How relevant {skill.name} is to the detected stack.",
|
|
121
|
+
))
|
|
122
|
+
|
|
123
|
+
# ── 4. Instruction compression ratio ────────────────────────────
|
|
124
|
+
if skill.instructions:
|
|
125
|
+
raw_lines = len(skill.instructions.splitlines())
|
|
126
|
+
optimizer = TokenOptimizer()
|
|
127
|
+
compressed = optimizer.compress_instructions([skill.instructions])
|
|
128
|
+
compressed_lines = len(compressed.splitlines())
|
|
129
|
+
ratio = ((raw_lines - compressed_lines) / raw_lines * 100) if raw_lines > 0 else 0
|
|
130
|
+
metrics.append(BenchmarkMetric(
|
|
131
|
+
name="Instruction Compression",
|
|
132
|
+
before=raw_lines,
|
|
133
|
+
after=compressed_lines,
|
|
134
|
+
unit="lines",
|
|
135
|
+
improvement_pct=round(max(0, ratio), 1),
|
|
136
|
+
description="Compression achieved on skill instructions.",
|
|
137
|
+
))
|
|
138
|
+
|
|
139
|
+
# ── 5. Verification: does the skill have all required fields? ────
|
|
140
|
+
verified = bool(
|
|
141
|
+
skill.instructions
|
|
142
|
+
and skill.name
|
|
143
|
+
and skill.description
|
|
144
|
+
and skill.stacks
|
|
145
|
+
)
|
|
146
|
+
|
|
147
|
+
return SkillBenchmark(
|
|
148
|
+
skill_id=skill.id,
|
|
149
|
+
skill_name=skill.name,
|
|
150
|
+
metrics=metrics,
|
|
151
|
+
verification_passed=verified,
|
|
152
|
+
verification_details=(
|
|
153
|
+
"All required fields present." if verified
|
|
154
|
+
else "Missing required fields: "
|
|
155
|
+
+ ", ".join(
|
|
156
|
+
f for f, v in [
|
|
157
|
+
("instructions", skill.instructions),
|
|
158
|
+
("name", skill.name),
|
|
159
|
+
("description", skill.description),
|
|
160
|
+
("stacks", skill.stacks),
|
|
161
|
+
] if not v
|
|
162
|
+
)
|
|
163
|
+
),
|
|
164
|
+
)
|
|
165
|
+
|
|
166
|
+
def _benchmark_governance(self, result) -> SkillBenchmark:
|
|
167
|
+
"""Benchmark the governance engine itself."""
|
|
168
|
+
total_policies = len(ALL_POLICIES)
|
|
169
|
+
violations = len(result.violations)
|
|
170
|
+
|
|
171
|
+
metrics = [
|
|
172
|
+
BenchmarkMetric(
|
|
173
|
+
name="Total Policies Active",
|
|
174
|
+
before=0,
|
|
175
|
+
after=total_policies,
|
|
176
|
+
unit="policies",
|
|
177
|
+
improvement_pct=100,
|
|
178
|
+
description="Number of security policies enforced.",
|
|
179
|
+
),
|
|
180
|
+
BenchmarkMetric(
|
|
181
|
+
name="Violations Detected",
|
|
182
|
+
before=0,
|
|
183
|
+
after=violations,
|
|
184
|
+
unit="violations",
|
|
185
|
+
improvement_pct=100 if violations > 0 else 0,
|
|
186
|
+
description="Policy violations caught by governance engine.",
|
|
187
|
+
),
|
|
188
|
+
BenchmarkMetric(
|
|
189
|
+
name="Risk Score",
|
|
190
|
+
before=100, # Assume worst case without governance
|
|
191
|
+
after=result.risk_score,
|
|
192
|
+
unit="score",
|
|
193
|
+
improvement_pct=round(max(0, (100 - result.risk_score)), 1),
|
|
194
|
+
description="Risk score (lower is better).",
|
|
195
|
+
),
|
|
196
|
+
BenchmarkMetric(
|
|
197
|
+
name="Compliance Coverage",
|
|
198
|
+
before=0,
|
|
199
|
+
after=len({fw for p in ALL_POLICIES for fw in p.compliance}),
|
|
200
|
+
unit="frameworks",
|
|
201
|
+
improvement_pct=100,
|
|
202
|
+
description="Number of compliance frameworks covered.",
|
|
203
|
+
),
|
|
204
|
+
]
|
|
205
|
+
|
|
206
|
+
return SkillBenchmark(
|
|
207
|
+
skill_id="governance-engine",
|
|
208
|
+
skill_name="Security Governance Engine",
|
|
209
|
+
metrics=metrics,
|
|
210
|
+
verification_passed=True,
|
|
211
|
+
verification_details=f"Governance engine operational. {total_policies} policies loaded.",
|
|
212
|
+
)
|
|
213
|
+
|
|
214
|
+
def _benchmark_optimization(self, opt_result: OptimizationResult) -> SkillBenchmark:
|
|
215
|
+
"""Benchmark the token optimization engine."""
|
|
216
|
+
metrics = [
|
|
217
|
+
BenchmarkMetric(
|
|
218
|
+
name="Token Reduction",
|
|
219
|
+
before=opt_result.original_tokens,
|
|
220
|
+
after=opt_result.optimized_tokens,
|
|
221
|
+
unit="tokens",
|
|
222
|
+
improvement_pct=opt_result.reduction_pct,
|
|
223
|
+
description="Tokens saved through optimization.",
|
|
224
|
+
),
|
|
225
|
+
BenchmarkMetric(
|
|
226
|
+
name="Rules Included",
|
|
227
|
+
before=opt_result.rules_included + opt_result.rules_excluded,
|
|
228
|
+
after=opt_result.rules_included,
|
|
229
|
+
unit="rules",
|
|
230
|
+
improvement_pct=round(
|
|
231
|
+
opt_result.rules_excluded / max(1, opt_result.rules_included + opt_result.rules_excluded) * 100, 1
|
|
232
|
+
),
|
|
233
|
+
description="Low-priority rules excluded to save tokens.",
|
|
234
|
+
),
|
|
235
|
+
]
|
|
236
|
+
|
|
237
|
+
return SkillBenchmark(
|
|
238
|
+
skill_id="optimization-engine",
|
|
239
|
+
skill_name="Token Optimization Engine",
|
|
240
|
+
metrics=metrics,
|
|
241
|
+
verification_passed=True,
|
|
242
|
+
verification_details=f"Optimization engine operational. {opt_result.reduction_pct:.1f}% token reduction.",
|
|
243
|
+
)
|
agentra/cli/__init__.py
ADDED
|
File without changes
|