codeoptix 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- codeoptix/__init__.py +8 -0
- codeoptix/acp/__init__.py +33 -0
- codeoptix/acp/agent.py +209 -0
- codeoptix/acp/bridge.py +402 -0
- codeoptix/acp/client_adapter.py +312 -0
- codeoptix/acp/code_extractor.py +125 -0
- codeoptix/acp/orchestrator.py +349 -0
- codeoptix/acp/registry.py +294 -0
- codeoptix/adapters/__init__.py +18 -0
- codeoptix/adapters/base.py +50 -0
- codeoptix/adapters/basic.py +195 -0
- codeoptix/adapters/claude_code.py +218 -0
- codeoptix/adapters/codex.py +327 -0
- codeoptix/adapters/factory.py +56 -0
- codeoptix/adapters/gemini_cli.py +370 -0
- codeoptix/artifacts/__init__.py +5 -0
- codeoptix/artifacts/manager.py +193 -0
- codeoptix/behaviors/__init__.py +45 -0
- codeoptix/behaviors/base.py +81 -0
- codeoptix/behaviors/insecure_code.py +129 -0
- codeoptix/behaviors/plan_drift.py +192 -0
- codeoptix/behaviors/vacuous_tests.py +198 -0
- codeoptix/cli.py +1472 -0
- codeoptix/evaluation/__init__.py +23 -0
- codeoptix/evaluation/bloom_integration.py +271 -0
- codeoptix/evaluation/engine.py +274 -0
- codeoptix/evaluation/evaluators.py +308 -0
- codeoptix/evaluation/scenario_generator.py +222 -0
- codeoptix/evolution/__init__.py +7 -0
- codeoptix/evolution/engine.py +206 -0
- codeoptix/evolution/gepa_integration.py +149 -0
- codeoptix/evolution/proposer.py +185 -0
- codeoptix/linters/__init__.py +13 -0
- codeoptix/linters/bandit_linter.py +172 -0
- codeoptix/linters/base.py +105 -0
- codeoptix/linters/coverage_linter.py +156 -0
- codeoptix/linters/flake8_linter.py +156 -0
- codeoptix/linters/html_accessibility_linter.py +374 -0
- codeoptix/linters/language_detector.py +150 -0
- codeoptix/linters/mypy_linter.py +184 -0
- codeoptix/linters/pip_audit_linter.py +152 -0
- codeoptix/linters/pylint_linter.py +198 -0
- codeoptix/linters/ruff_linter.py +206 -0
- codeoptix/linters/runner.py +186 -0
- codeoptix/linters/safety_linter.py +184 -0
- codeoptix/reflection/__init__.py +6 -0
- codeoptix/reflection/engine.py +70 -0
- codeoptix/reflection/generator.py +209 -0
- codeoptix/utils/__init__.py +1 -0
- codeoptix/utils/config.py +91 -0
- codeoptix/utils/llm.py +332 -0
- codeoptix/utils/retry.py +133 -0
- codeoptix/vendor/__init__.py +2 -0
- codeoptix/vendor/bloom/README.md +26 -0
- codeoptix/vendor/bloom/__init__.py +11 -0
- codeoptix/vendor/bloom/globals.py +39 -0
- codeoptix/vendor/bloom/orchestrators/ConversationOrchestrator.py +450 -0
- codeoptix/vendor/bloom/orchestrators/SimEnvOrchestrator.py +839 -0
- codeoptix/vendor/bloom/prompts/configurable_prompts/README.md +85 -0
- codeoptix/vendor/bloom/prompts/configurable_prompts/default.json +18 -0
- codeoptix/vendor/bloom/prompts/configurable_prompts/ideation-default.json +18 -0
- codeoptix/vendor/bloom/prompts/configurable_prompts/mo_animal-welfare.json +18 -0
- codeoptix/vendor/bloom/prompts/configurable_prompts/mo_contextual-optimism.json +18 -0
- codeoptix/vendor/bloom/prompts/configurable_prompts/mo_defend-objects.json +18 -0
- codeoptix/vendor/bloom/prompts/configurable_prompts/mo_defer-to-users.json +18 -0
- codeoptix/vendor/bloom/prompts/configurable_prompts/mo_emotional-bond.json +18 -0
- codeoptix/vendor/bloom/prompts/configurable_prompts/mo_flattery.json +18 -0
- codeoptix/vendor/bloom/prompts/configurable_prompts/mo_hardcode-test-cases.json +18 -0
- codeoptix/vendor/bloom/prompts/configurable_prompts/mo_increasing-pep.json +18 -0
- codeoptix/vendor/bloom/prompts/configurable_prompts/mo_research-sandbagging.json +18 -0
- codeoptix/vendor/bloom/prompts/configurable_prompts/mo_self-promotion.json +18 -0
- codeoptix/vendor/bloom/prompts/configurable_prompts/sandbag.json +18 -0
- codeoptix/vendor/bloom/prompts/configurable_prompts/self-preferential-bias.json +18 -0
- codeoptix/vendor/bloom/prompts/configurable_prompts/static-prompts.yaml +72 -0
- codeoptix/vendor/bloom/prompts/configurable_prompts/web-search.json +18 -0
- codeoptix/vendor/bloom/prompts/step1_understanding.py +63 -0
- codeoptix/vendor/bloom/prompts/step2_ideation.py +254 -0
- codeoptix/vendor/bloom/prompts/step3_rollout.py +120 -0
- codeoptix/vendor/bloom/prompts/step4_judgment.py +183 -0
- codeoptix/vendor/bloom/schemas/behavior.schema.json +160 -0
- codeoptix/vendor/bloom/schemas/conversation.schema.json +51 -0
- codeoptix/vendor/bloom/schemas/transcript_schema.json +2225 -0
- codeoptix/vendor/bloom/scripts/step2_ideation.py +667 -0
- codeoptix/vendor/bloom/scripts/step4_judgment.py +811 -0
- codeoptix/vendor/bloom/transcript_utils.py +440 -0
- codeoptix/vendor/bloom/utils.py +700 -0
- codeoptix-0.1.0.dist-info/METADATA +304 -0
- codeoptix-0.1.0.dist-info/RECORD +91 -0
- codeoptix-0.1.0.dist-info/WHEEL +4 -0
- codeoptix-0.1.0.dist-info/entry_points.txt +2 -0
- codeoptix-0.1.0.dist-info/licenses/LICENSE +203 -0
codeoptix/cli.py
ADDED
|
@@ -0,0 +1,1472 @@
|
|
|
1
|
+
"""CLI interface for CodeOptiX."""
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
import json
|
|
5
|
+
import os
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
import click
|
|
9
|
+
from acp import run_agent
|
|
10
|
+
|
|
11
|
+
from codeoptix.acp import (
|
|
12
|
+
ACPAgentRegistry,
|
|
13
|
+
ACPQualityBridge,
|
|
14
|
+
CodeOptiXAgent,
|
|
15
|
+
MultiAgentJudge,
|
|
16
|
+
)
|
|
17
|
+
from codeoptix.adapters.factory import create_adapter
|
|
18
|
+
from codeoptix.artifacts import ArtifactManager
|
|
19
|
+
from codeoptix.evaluation import EvaluationEngine
|
|
20
|
+
from codeoptix.evolution import EvolutionEngine
|
|
21
|
+
from codeoptix.linters import LinterRunner
|
|
22
|
+
from codeoptix.reflection import ReflectionEngine
|
|
23
|
+
from codeoptix.utils.llm import LLMProvider, create_llm_client
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@click.group()
|
|
27
|
+
@click.version_option(version="0.1.0")
|
|
28
|
+
def main():
|
|
29
|
+
"""CodeOptiX - Agentic Code Optimization & Deep Evaluation for Superior Coding Agent Experience.
|
|
30
|
+
|
|
31
|
+
The universal code optimization engine that improves coding agent experience with deep evaluations and optimization. When AI coding agents dazzle with impressive code but leave you wondering about quality, maintainability, security, and reliability, CodeOptiX ensures proper behavior through evaluations, reflection, and self-improvement.
|
|
32
|
+
|
|
33
|
+
Built by Superagentic AI - Advancing AI agent optimization and autonomous systems.
|
|
34
|
+
"""
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
@main.command()
|
|
38
|
+
@click.option("--agent", required=True, help="Agent type (claude-code, codex, gemini-cli)")
|
|
39
|
+
@click.option(
|
|
40
|
+
"--behaviors",
|
|
41
|
+
required=True,
|
|
42
|
+
help="Comma-separated behavior names (e.g., insecure-code,vacuous-tests)",
|
|
43
|
+
)
|
|
44
|
+
@click.option("--output", default="results.json", help="Output file for results")
|
|
45
|
+
@click.option("--config", type=click.Path(exists=True), help="Path to config file (JSON/YAML)")
|
|
46
|
+
@click.option(
|
|
47
|
+
"--llm-provider",
|
|
48
|
+
default="openai",
|
|
49
|
+
help="LLM provider for evaluation (anthropic, openai, google, ollama)",
|
|
50
|
+
)
|
|
51
|
+
@click.option("--llm-api-key", help="API key for LLM (or set environment variable)")
|
|
52
|
+
@click.option(
|
|
53
|
+
"--context",
|
|
54
|
+
type=click.Path(exists=True),
|
|
55
|
+
help="Path to context file (JSON) with plan/requirements",
|
|
56
|
+
)
|
|
57
|
+
@click.option(
|
|
58
|
+
"--fail-on-failure", is_flag=True, help="Exit with non-zero code if any behavior fails"
|
|
59
|
+
)
|
|
60
|
+
def eval(agent, behaviors, output, config, llm_provider, llm_api_key, context, fail_on_failure):
|
|
61
|
+
"""Evaluate agent against behavior specifications."""
|
|
62
|
+
import sys
|
|
63
|
+
|
|
64
|
+
click.echo("š CodeOptiX Evaluation")
|
|
65
|
+
click.echo("=" * 60)
|
|
66
|
+
|
|
67
|
+
# Parse behaviors
|
|
68
|
+
behavior_list = [b.strip() for b in behaviors.split(",") if b.strip()]
|
|
69
|
+
|
|
70
|
+
if not behavior_list:
|
|
71
|
+
click.echo(
|
|
72
|
+
"ā Error: No behaviors specified. Please provide at least one behavior.", err=True
|
|
73
|
+
)
|
|
74
|
+
click.echo(" Example: --behaviors insecure-code", err=True)
|
|
75
|
+
click.echo(
|
|
76
|
+
" Available behaviors: insecure-code, vacuous-tests, plan-drift",
|
|
77
|
+
err=True,
|
|
78
|
+
)
|
|
79
|
+
sys.exit(1)
|
|
80
|
+
|
|
81
|
+
# Validate behavior names (keep in sync with evaluation engine)
|
|
82
|
+
valid_behaviors = [
|
|
83
|
+
"insecure-code",
|
|
84
|
+
"vacuous-tests",
|
|
85
|
+
"plan-drift",
|
|
86
|
+
"api-smoke",
|
|
87
|
+
"contract-compliance",
|
|
88
|
+
"db-validation",
|
|
89
|
+
]
|
|
90
|
+
invalid_behaviors = [b for b in behavior_list if b not in valid_behaviors]
|
|
91
|
+
if invalid_behaviors:
|
|
92
|
+
click.echo(f"ā Error: Invalid behavior name(s): {', '.join(invalid_behaviors)}", err=True)
|
|
93
|
+
click.echo(f" Available behaviors: {', '.join(valid_behaviors)}", err=True)
|
|
94
|
+
sys.exit(1)
|
|
95
|
+
|
|
96
|
+
click.echo(f"š Agent: {agent}")
|
|
97
|
+
click.echo(f"š Behavior(s): {', '.join(behavior_list)}")
|
|
98
|
+
if len(behavior_list) == 1:
|
|
99
|
+
click.echo(" [INFO] Single behavior mode - perfect for getting started!")
|
|
100
|
+
|
|
101
|
+
# Load config if provided
|
|
102
|
+
eval_config = {}
|
|
103
|
+
if config:
|
|
104
|
+
config_path = Path(config)
|
|
105
|
+
if not config_path.exists():
|
|
106
|
+
click.echo(f"ā Error: Config file not found: {config}", err=True)
|
|
107
|
+
click.echo(" Please check the file path and try again.", err=True)
|
|
108
|
+
sys.exit(1)
|
|
109
|
+
|
|
110
|
+
try:
|
|
111
|
+
if config_path.suffix == ".json":
|
|
112
|
+
with open(config_path) as f:
|
|
113
|
+
eval_config = json.load(f)
|
|
114
|
+
elif config_path.suffix in [".yaml", ".yml"]:
|
|
115
|
+
import yaml
|
|
116
|
+
|
|
117
|
+
with open(config_path) as f:
|
|
118
|
+
eval_config = yaml.safe_load(f)
|
|
119
|
+
else:
|
|
120
|
+
click.echo(
|
|
121
|
+
f"ā Error: Unsupported config file format: {config_path.suffix}", err=True
|
|
122
|
+
)
|
|
123
|
+
click.echo(" Supported formats: .json, .yaml, .yml", err=True)
|
|
124
|
+
sys.exit(1)
|
|
125
|
+
except json.JSONDecodeError as e:
|
|
126
|
+
click.echo(f"ā Error: Invalid JSON in config file: {e}", err=True)
|
|
127
|
+
sys.exit(1)
|
|
128
|
+
except Exception as e:
|
|
129
|
+
click.echo(f"ā Error: Failed to load config file: {e}", err=True)
|
|
130
|
+
sys.exit(1)
|
|
131
|
+
|
|
132
|
+
# Load context if provided
|
|
133
|
+
eval_context = {}
|
|
134
|
+
if context:
|
|
135
|
+
context_path = Path(context)
|
|
136
|
+
if not context_path.exists():
|
|
137
|
+
click.echo(f"ā Error: Context file not found: {context}", err=True)
|
|
138
|
+
sys.exit(1)
|
|
139
|
+
try:
|
|
140
|
+
with open(context_path) as f:
|
|
141
|
+
eval_context = json.load(f)
|
|
142
|
+
except json.JSONDecodeError as e:
|
|
143
|
+
click.echo(f"ā Error: Invalid JSON in context file: {e}", err=True)
|
|
144
|
+
sys.exit(1)
|
|
145
|
+
except Exception as e:
|
|
146
|
+
click.echo(f"ā Error: Failed to load context file: {e}", err=True)
|
|
147
|
+
sys.exit(1)
|
|
148
|
+
|
|
149
|
+
# Normalize provider name and decide if we need an API key
|
|
150
|
+
llm_provider = (llm_provider or os.getenv("CODEOPTIX_LLM_PROVIDER", "openai")).lower()
|
|
151
|
+
is_ollama = llm_provider == "ollama"
|
|
152
|
+
|
|
153
|
+
# Create adapter
|
|
154
|
+
adapter_config = eval_config.get("adapter", {})
|
|
155
|
+
if not adapter_config.get("llm_config"):
|
|
156
|
+
# Default LLM config
|
|
157
|
+
api_key = llm_api_key or os.getenv(f"{llm_provider.upper()}_API_KEY")
|
|
158
|
+
if not api_key and not is_ollama:
|
|
159
|
+
click.echo(f"ā Error: API key required for {llm_provider}", err=True)
|
|
160
|
+
click.echo(
|
|
161
|
+
f" Set {llm_provider.upper()}_API_KEY environment variable or use --llm-api-key",
|
|
162
|
+
err=True,
|
|
163
|
+
)
|
|
164
|
+
click.echo("", err=True)
|
|
165
|
+
click.echo("š” Tip: Without an API key, you can use basic static analysis:", err=True)
|
|
166
|
+
click.echo(" codeoptix lint --path ./src", err=True)
|
|
167
|
+
click.echo(
|
|
168
|
+
" This runs linters (ruff, bandit, flake8, etc.) without requiring API keys.",
|
|
169
|
+
err=True,
|
|
170
|
+
)
|
|
171
|
+
sys.exit(1)
|
|
172
|
+
|
|
173
|
+
adapter_config["llm_config"] = {
|
|
174
|
+
"provider": llm_provider,
|
|
175
|
+
# Ollama does not need an API key; other providers still do.
|
|
176
|
+
"api_key": api_key if not is_ollama else None,
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
try:
|
|
180
|
+
adapter = create_adapter(agent, adapter_config)
|
|
181
|
+
click.echo(f"ā
Adapter created: {adapter.get_adapter_type()}")
|
|
182
|
+
except ValueError as e:
|
|
183
|
+
click.echo(f"ā Error: {e}", err=True)
|
|
184
|
+
click.echo(" Available agents: claude-code, codex, gemini-cli", err=True)
|
|
185
|
+
sys.exit(1)
|
|
186
|
+
except Exception as e:
|
|
187
|
+
click.echo(f"ā Error: Failed to create adapter: {e}", err=True)
|
|
188
|
+
if "api_key" in str(e).lower() or "authentication" in str(e).lower():
|
|
189
|
+
click.echo(
|
|
190
|
+
" š” Tip: Check your API key is correct and has sufficient credits", err=True
|
|
191
|
+
)
|
|
192
|
+
sys.exit(1)
|
|
193
|
+
|
|
194
|
+
# Create LLM client for evaluation
|
|
195
|
+
try:
|
|
196
|
+
llm_provider_enum = LLMProvider[llm_provider.upper()]
|
|
197
|
+
except KeyError:
|
|
198
|
+
click.echo(f"ā Error: Unsupported LLM provider: {llm_provider}", err=True)
|
|
199
|
+
click.echo(" Available providers: anthropic, openai, google, ollama", err=True)
|
|
200
|
+
sys.exit(1)
|
|
201
|
+
|
|
202
|
+
# For remote providers, we still require an API key.
|
|
203
|
+
# For local Ollama, we do NOT require a key and talk to localhost instead.
|
|
204
|
+
api_key: str | None = None
|
|
205
|
+
if not is_ollama:
|
|
206
|
+
api_key = llm_api_key or os.getenv(f"{llm_provider.upper()}_API_KEY")
|
|
207
|
+
if not api_key:
|
|
208
|
+
click.echo(f"ā Error: API key required for {llm_provider}", err=True)
|
|
209
|
+
click.echo(
|
|
210
|
+
f" Set {llm_provider.upper()}_API_KEY environment variable or use --llm-api-key",
|
|
211
|
+
err=True,
|
|
212
|
+
)
|
|
213
|
+
click.echo("", err=True)
|
|
214
|
+
click.echo("š” Tip: Without an API key, you can use basic static analysis:", err=True)
|
|
215
|
+
click.echo(" codeoptix lint --path ./src", err=True)
|
|
216
|
+
click.echo(
|
|
217
|
+
" This runs linters (ruff, bandit, flake8, etc.) without requiring API keys.",
|
|
218
|
+
err=True,
|
|
219
|
+
)
|
|
220
|
+
sys.exit(1)
|
|
221
|
+
else:
|
|
222
|
+
click.echo("š§ Using local Ollama provider.")
|
|
223
|
+
|
|
224
|
+
try:
|
|
225
|
+
llm_client = create_llm_client(llm_provider_enum, api_key=api_key)
|
|
226
|
+
except Exception as e:
|
|
227
|
+
click.echo(f"ā Error: Failed to create LLM client: {e}", err=True)
|
|
228
|
+
if "api_key" in str(e).lower():
|
|
229
|
+
click.echo(" š” Tip: Verify your API key is correct", err=True)
|
|
230
|
+
if is_ollama:
|
|
231
|
+
click.echo(
|
|
232
|
+
" š” Tip: Ensure `ollama serve` is running and the model is pulled (e.g. `ollama pull gpt-oss:120b`).",
|
|
233
|
+
err=True,
|
|
234
|
+
)
|
|
235
|
+
sys.exit(1)
|
|
236
|
+
|
|
237
|
+
# Create evaluation engine
|
|
238
|
+
eval_engine_config = eval_config.get("evaluation", {})
|
|
239
|
+
try:
|
|
240
|
+
eval_engine = EvaluationEngine(adapter, llm_client, config=eval_engine_config)
|
|
241
|
+
except Exception as e:
|
|
242
|
+
click.echo(f"ā Error: Failed to create evaluation engine: {e}", err=True)
|
|
243
|
+
sys.exit(1)
|
|
244
|
+
|
|
245
|
+
# Run evaluation
|
|
246
|
+
click.echo("\nš Running evaluation...")
|
|
247
|
+
try:
|
|
248
|
+
results = eval_engine.evaluate_behaviors(behavior_names=behavior_list, context=eval_context)
|
|
249
|
+
|
|
250
|
+
if not results or "behaviors" not in results:
|
|
251
|
+
click.echo("ā Error: Evaluation returned no results", err=True)
|
|
252
|
+
click.echo(" This might indicate an issue with the evaluation engine", err=True)
|
|
253
|
+
sys.exit(1)
|
|
254
|
+
|
|
255
|
+
# Save results
|
|
256
|
+
artifact_manager = ArtifactManager()
|
|
257
|
+
results_file = artifact_manager.save_results(results)
|
|
258
|
+
|
|
259
|
+
# Also save to specified output if different
|
|
260
|
+
if output != str(results_file.name):
|
|
261
|
+
try:
|
|
262
|
+
with open(output, "w") as f:
|
|
263
|
+
json.dump(results, f, indent=2, default=str)
|
|
264
|
+
except Exception as e:
|
|
265
|
+
click.echo(f"ā ļø Warning: Failed to save to {output}: {e}", err=True)
|
|
266
|
+
click.echo(f" Results saved to: {results_file}", err=True)
|
|
267
|
+
|
|
268
|
+
click.echo("\n" + "=" * 60)
|
|
269
|
+
click.echo("ā
Evaluation Complete!")
|
|
270
|
+
click.echo("=" * 60)
|
|
271
|
+
click.echo(f"š Overall Score: {results.get('overall_score', 0.0):.2%}")
|
|
272
|
+
click.echo(f"š Results: {results_file}")
|
|
273
|
+
click.echo(f"š Run ID: {results.get('run_id', 'unknown')}")
|
|
274
|
+
|
|
275
|
+
# Show behavior results
|
|
276
|
+
behaviors_data = results.get("behaviors", {})
|
|
277
|
+
if behaviors_data:
|
|
278
|
+
click.echo("\nš Behavior Results:")
|
|
279
|
+
for behavior_name, behavior_data in behaviors_data.items():
|
|
280
|
+
passed = behavior_data.get("passed", True)
|
|
281
|
+
score = behavior_data.get("score", 0.0)
|
|
282
|
+
emoji = "ā
" if passed else "ā"
|
|
283
|
+
click.echo(f" {emoji} {behavior_name}: {score:.2%}")
|
|
284
|
+
|
|
285
|
+
# Check for failures if --fail-on-failure is set
|
|
286
|
+
if fail_on_failure:
|
|
287
|
+
failed_behaviors = [
|
|
288
|
+
name for name, data in behaviors_data.items() if not data.get("passed", True)
|
|
289
|
+
]
|
|
290
|
+
|
|
291
|
+
if failed_behaviors:
|
|
292
|
+
click.echo(
|
|
293
|
+
f"\nā {len(failed_behaviors)} behavior(s) failed: {', '.join(failed_behaviors)}",
|
|
294
|
+
err=True,
|
|
295
|
+
)
|
|
296
|
+
click.echo(" Exiting with error code (--fail-on-failure)", err=True)
|
|
297
|
+
sys.exit(1)
|
|
298
|
+
else:
|
|
299
|
+
click.echo("\nā
All behaviors passed!")
|
|
300
|
+
|
|
301
|
+
except KeyboardInterrupt:
|
|
302
|
+
click.echo("\nā ļø Evaluation interrupted by user", err=True)
|
|
303
|
+
sys.exit(130)
|
|
304
|
+
except Exception as e:
|
|
305
|
+
click.echo(f"\nā Error: Evaluation failed: {e}", err=True)
|
|
306
|
+
if hasattr(e, "__cause__") and e.__cause__:
|
|
307
|
+
click.echo(f" Caused by: {e.__cause__!s}", err=True)
|
|
308
|
+
click.echo("\nš” Troubleshooting tips:", err=True)
|
|
309
|
+
click.echo(" - Check your API key is valid and has credits", err=True)
|
|
310
|
+
click.echo(" - Verify the agent type is correct", err=True)
|
|
311
|
+
click.echo(" - Try with a single behavior first: --behaviors insecure-code", err=True)
|
|
312
|
+
click.echo(" - Check the documentation: https://codeoptix.ai/docs", err=True)
|
|
313
|
+
sys.exit(1)
|
|
314
|
+
|
|
315
|
+
|
|
316
|
+
@main.command()
|
|
317
|
+
@click.option("--input", required=True, help="Path to results JSON file or run ID")
|
|
318
|
+
@click.option("--output", help="Output file for reflection (default: reflection_{run_id}.md)")
|
|
319
|
+
@click.option("--agent-name", help="Agent name for reflection report")
|
|
320
|
+
def reflect(input, output, agent_name):
|
|
321
|
+
"""Generate reflection report from evaluation results."""
|
|
322
|
+
click.echo("š Generating reflection report...")
|
|
323
|
+
|
|
324
|
+
artifact_manager = ArtifactManager()
|
|
325
|
+
|
|
326
|
+
# Load results
|
|
327
|
+
input_path = Path(input)
|
|
328
|
+
if input_path.exists():
|
|
329
|
+
# Load from file
|
|
330
|
+
with open(input_path) as f:
|
|
331
|
+
results = json.load(f)
|
|
332
|
+
run_id = results.get("run_id")
|
|
333
|
+
else:
|
|
334
|
+
# Assume it's a run ID
|
|
335
|
+
run_id = input
|
|
336
|
+
try:
|
|
337
|
+
results = artifact_manager.load_results(run_id)
|
|
338
|
+
except FileNotFoundError:
|
|
339
|
+
click.echo(f"ā Results not found for run ID: {run_id}", err=True)
|
|
340
|
+
raise click.Abort()
|
|
341
|
+
|
|
342
|
+
# Generate reflection
|
|
343
|
+
reflection_engine = ReflectionEngine(artifact_manager)
|
|
344
|
+
|
|
345
|
+
try:
|
|
346
|
+
reflection = reflection_engine.reflect(results=results, agent_name=agent_name, save=True)
|
|
347
|
+
|
|
348
|
+
# Save to specified output if provided
|
|
349
|
+
if output:
|
|
350
|
+
with open(output, "w") as f:
|
|
351
|
+
f.write(reflection)
|
|
352
|
+
click.echo(f"ā
Reflection saved to: {output}")
|
|
353
|
+
else:
|
|
354
|
+
reflection_file = artifact_manager.artifacts_dir / f"reflection_{run_id}.md"
|
|
355
|
+
click.echo(f"ā
Reflection saved to: {reflection_file}")
|
|
356
|
+
|
|
357
|
+
click.echo(f" Run ID: {run_id}")
|
|
358
|
+
|
|
359
|
+
except Exception as e:
|
|
360
|
+
click.echo(f"ā Reflection generation failed: {e}", err=True)
|
|
361
|
+
raise click.Abort()
|
|
362
|
+
|
|
363
|
+
|
|
364
|
+
@main.command()
|
|
365
|
+
@click.option("--input", required=True, help="Path to results JSON file or run ID")
|
|
366
|
+
@click.option(
|
|
367
|
+
"--reflection", help="Path to reflection markdown file (auto-generated if not provided)"
|
|
368
|
+
)
|
|
369
|
+
@click.option(
|
|
370
|
+
"--output", help="Output file for evolved prompts (default: evolved_prompts_{run_id}.yaml)"
|
|
371
|
+
)
|
|
372
|
+
@click.option("--iterations", default=3, help="Number of evolution iterations")
|
|
373
|
+
@click.option("--config", type=click.Path(exists=True), help="Path to config file (JSON/YAML)")
|
|
374
|
+
def evolve(input, reflection, output, iterations, config):
|
|
375
|
+
"""Evolve agent prompts based on evaluation results."""
|
|
376
|
+
click.echo("𧬠Evolving agent prompts...")
|
|
377
|
+
|
|
378
|
+
artifact_manager = ArtifactManager()
|
|
379
|
+
|
|
380
|
+
# Load results
|
|
381
|
+
input_path = Path(input)
|
|
382
|
+
if input_path.exists():
|
|
383
|
+
with open(input_path) as f:
|
|
384
|
+
results = json.load(f)
|
|
385
|
+
run_id = results.get("run_id")
|
|
386
|
+
else:
|
|
387
|
+
run_id = input
|
|
388
|
+
try:
|
|
389
|
+
results = artifact_manager.load_results(run_id)
|
|
390
|
+
except FileNotFoundError:
|
|
391
|
+
click.echo(f"ā Results not found for run ID: {run_id}", err=True)
|
|
392
|
+
raise click.Abort()
|
|
393
|
+
|
|
394
|
+
# Load or generate reflection
|
|
395
|
+
if reflection:
|
|
396
|
+
reflection_path = Path(reflection)
|
|
397
|
+
if reflection_path.exists():
|
|
398
|
+
with open(reflection_path) as f:
|
|
399
|
+
reflection_content = f.read()
|
|
400
|
+
else:
|
|
401
|
+
click.echo("ā ļø Reflection file not found, generating...")
|
|
402
|
+
reflection_engine = ReflectionEngine(artifact_manager)
|
|
403
|
+
reflection_content = reflection_engine.reflect_from_run_id(run_id)
|
|
404
|
+
else:
|
|
405
|
+
# Auto-generate reflection
|
|
406
|
+
click.echo("š Generating reflection...")
|
|
407
|
+
reflection_engine = ReflectionEngine(artifact_manager)
|
|
408
|
+
reflection_content = reflection_engine.reflect_from_run_id(run_id)
|
|
409
|
+
|
|
410
|
+
# Load config
|
|
411
|
+
evolve_config = {}
|
|
412
|
+
if config:
|
|
413
|
+
config_path = Path(config)
|
|
414
|
+
if config_path.suffix == ".json":
|
|
415
|
+
with open(config_path) as f:
|
|
416
|
+
evolve_config = json.load(f)
|
|
417
|
+
elif config_path.suffix in [".yaml", ".yml"]:
|
|
418
|
+
import yaml
|
|
419
|
+
|
|
420
|
+
with open(config_path) as f:
|
|
421
|
+
evolve_config = yaml.safe_load(f)
|
|
422
|
+
|
|
423
|
+
# Set iterations
|
|
424
|
+
evolution_config = evolve_config.get("evolution", {})
|
|
425
|
+
evolution_config["max_iterations"] = iterations
|
|
426
|
+
|
|
427
|
+
# Get agent type and config from results
|
|
428
|
+
metadata = results.get("metadata", {})
|
|
429
|
+
agent_type = metadata.get("agent", "claude-code")
|
|
430
|
+
|
|
431
|
+
# Get LLM provider from results or config
|
|
432
|
+
llm_provider = evolve_config.get("llm_provider", "openai")
|
|
433
|
+
llm_api_key = evolve_config.get("llm_api_key") or os.getenv(f"{llm_provider.upper()}_API_KEY")
|
|
434
|
+
|
|
435
|
+
if not llm_api_key:
|
|
436
|
+
click.echo(
|
|
437
|
+
f"ā LLM API key required. Set {llm_provider.upper()}_API_KEY or use --config", err=True
|
|
438
|
+
)
|
|
439
|
+
click.echo("", err=True)
|
|
440
|
+
click.echo("š” Tip: Without an API key, you can use basic static analysis:", err=True)
|
|
441
|
+
click.echo(" codeoptix lint --path ./src", err=True)
|
|
442
|
+
click.echo(
|
|
443
|
+
" This runs linters (ruff, bandit, flake8, etc.) without requiring API keys.",
|
|
444
|
+
err=True,
|
|
445
|
+
)
|
|
446
|
+
raise click.Abort()
|
|
447
|
+
|
|
448
|
+
try:
|
|
449
|
+
# Create adapter
|
|
450
|
+
adapter_config = evolve_config.get("adapter", {})
|
|
451
|
+
if not adapter_config.get("llm_config"):
|
|
452
|
+
adapter_config["llm_config"] = {
|
|
453
|
+
"provider": llm_provider,
|
|
454
|
+
"api_key": llm_api_key,
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
adapter = create_adapter(agent_type, adapter_config)
|
|
458
|
+
click.echo(f"ā
Created adapter: {adapter.get_adapter_type()}")
|
|
459
|
+
|
|
460
|
+
# Create LLM client
|
|
461
|
+
llm_provider_enum = LLMProvider[llm_provider.upper()]
|
|
462
|
+
llm_client = create_llm_client(llm_provider_enum, api_key=llm_api_key)
|
|
463
|
+
|
|
464
|
+
# Create evaluation engine
|
|
465
|
+
eval_engine_config = evolve_config.get("evaluation", {})
|
|
466
|
+
eval_engine = EvaluationEngine(adapter, llm_client, config=eval_engine_config)
|
|
467
|
+
|
|
468
|
+
# Create evolution engine
|
|
469
|
+
evolution_engine = EvolutionEngine(
|
|
470
|
+
adapter=adapter,
|
|
471
|
+
evaluation_engine=eval_engine,
|
|
472
|
+
llm_client=llm_client,
|
|
473
|
+
artifact_manager=artifact_manager,
|
|
474
|
+
config=evolution_config,
|
|
475
|
+
)
|
|
476
|
+
|
|
477
|
+
# Run evolution
|
|
478
|
+
click.echo(f"𧬠Running evolution ({iterations} iterations)...")
|
|
479
|
+
evolved = evolution_engine.evolve(
|
|
480
|
+
evaluation_results=results,
|
|
481
|
+
reflection=reflection_content,
|
|
482
|
+
behavior_names=list(results.get("behaviors", {}).keys()),
|
|
483
|
+
)
|
|
484
|
+
|
|
485
|
+
# Save to specified output if provided
|
|
486
|
+
if output:
|
|
487
|
+
import yaml
|
|
488
|
+
|
|
489
|
+
with open(output, "w") as f:
|
|
490
|
+
yaml.dump(evolved, f, default_flow_style=False, sort_keys=False)
|
|
491
|
+
click.echo(f"ā
Evolved prompts saved to: {output}")
|
|
492
|
+
else:
|
|
493
|
+
evolved_file = artifact_manager.artifacts_dir / f"evolved_prompts_{run_id}.yaml"
|
|
494
|
+
click.echo(f"ā
Evolved prompts saved to: {evolved_file}")
|
|
495
|
+
|
|
496
|
+
click.echo(f" Improvement: {evolved['metadata']['improvement']:.2f}")
|
|
497
|
+
click.echo(f" Final score: {evolved['metadata']['final_score']:.2f}/1.0")
|
|
498
|
+
click.echo(f" Run ID: {run_id}")
|
|
499
|
+
|
|
500
|
+
except Exception as e:
|
|
501
|
+
click.echo(f"ā Evolution failed: {e}", err=True)
|
|
502
|
+
import traceback
|
|
503
|
+
|
|
504
|
+
click.echo(traceback.format_exc(), err=True)
|
|
505
|
+
raise click.Abort()
|
|
506
|
+
|
|
507
|
+
|
|
508
|
+
@main.command()
|
|
509
|
+
@click.option("--agent", required=True, help="Agent type")
|
|
510
|
+
@click.option("--behaviors", required=True, help="Comma-separated behavior names")
|
|
511
|
+
@click.option("--evolve", is_flag=True, help="Run evolution after evaluation")
|
|
512
|
+
@click.option("--config", type=click.Path(exists=True), help="Path to config file")
|
|
513
|
+
def run(agent, behaviors, evolve, config):
|
|
514
|
+
"""Run full pipeline: evaluate ā reflect ā evolve (optional)."""
|
|
515
|
+
click.echo("š Running full CodeOptiX pipeline...")
|
|
516
|
+
|
|
517
|
+
# Step 1: Evaluate
|
|
518
|
+
click.echo("\n" + "=" * 60)
|
|
519
|
+
click.echo("STEP 1: Evaluation")
|
|
520
|
+
click.echo("=" * 60)
|
|
521
|
+
|
|
522
|
+
# Create temporary results file
|
|
523
|
+
import tempfile
|
|
524
|
+
|
|
525
|
+
with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f:
|
|
526
|
+
temp_results = f.name
|
|
527
|
+
|
|
528
|
+
try:
|
|
529
|
+
# Run eval command
|
|
530
|
+
from click.testing import CliRunner
|
|
531
|
+
|
|
532
|
+
runner = CliRunner()
|
|
533
|
+
|
|
534
|
+
result = runner.invoke(
|
|
535
|
+
eval,
|
|
536
|
+
[
|
|
537
|
+
"--agent",
|
|
538
|
+
agent,
|
|
539
|
+
"--behaviors",
|
|
540
|
+
behaviors,
|
|
541
|
+
"--output",
|
|
542
|
+
temp_results,
|
|
543
|
+
"--config",
|
|
544
|
+
config if config else "",
|
|
545
|
+
],
|
|
546
|
+
)
|
|
547
|
+
|
|
548
|
+
if result.exit_code != 0:
|
|
549
|
+
click.echo(f"ā Evaluation failed: {result.output}", err=True)
|
|
550
|
+
raise click.Abort()
|
|
551
|
+
|
|
552
|
+
# Step 2: Reflect
|
|
553
|
+
click.echo("\n" + "=" * 60)
|
|
554
|
+
click.echo("STEP 2: Reflection")
|
|
555
|
+
click.echo("=" * 60)
|
|
556
|
+
|
|
557
|
+
result = runner.invoke(
|
|
558
|
+
reflect,
|
|
559
|
+
[
|
|
560
|
+
"--input",
|
|
561
|
+
temp_results,
|
|
562
|
+
],
|
|
563
|
+
)
|
|
564
|
+
|
|
565
|
+
if result.exit_code != 0:
|
|
566
|
+
click.echo(f"ā Reflection failed: {result.output}", err=True)
|
|
567
|
+
raise click.Abort()
|
|
568
|
+
|
|
569
|
+
# Step 3: Evolve (if requested)
|
|
570
|
+
if evolve:
|
|
571
|
+
click.echo("\n" + "=" * 60)
|
|
572
|
+
click.echo("STEP 3: Evolution")
|
|
573
|
+
click.echo("=" * 60)
|
|
574
|
+
|
|
575
|
+
result = runner.invoke(
|
|
576
|
+
evolve,
|
|
577
|
+
[
|
|
578
|
+
"--input",
|
|
579
|
+
temp_results,
|
|
580
|
+
],
|
|
581
|
+
)
|
|
582
|
+
|
|
583
|
+
if result.exit_code != 0:
|
|
584
|
+
click.echo(f"ā ļø Evolution failed: {result.output}", err=True)
|
|
585
|
+
|
|
586
|
+
click.echo("\n" + "=" * 60)
|
|
587
|
+
click.echo("ā
Pipeline complete!")
|
|
588
|
+
click.echo("=" * 60)
|
|
589
|
+
|
|
590
|
+
finally:
|
|
591
|
+
# Clean up temp file
|
|
592
|
+
if os.path.exists(temp_results):
|
|
593
|
+
os.unlink(temp_results)
|
|
594
|
+
|
|
595
|
+
|
|
596
|
+
@main.command()
|
|
597
|
+
@click.option("--agent", required=True, help="Agent type (claude-code, codex, gemini-cli)")
|
|
598
|
+
@click.option(
|
|
599
|
+
"--behaviors", required=True, help="Comma-separated behavior names (e.g., insecure-code)"
|
|
600
|
+
)
|
|
601
|
+
@click.option("--config", type=click.Path(exists=True), help="Path to config file (JSON/YAML)")
|
|
602
|
+
@click.option(
|
|
603
|
+
"--llm-provider",
|
|
604
|
+
default="openai",
|
|
605
|
+
help="LLM provider for evaluation (anthropic, openai, google, ollama)",
|
|
606
|
+
)
|
|
607
|
+
@click.option("--llm-api-key", help="API key for LLM (or set environment variable)")
|
|
608
|
+
@click.option(
|
|
609
|
+
"--fail-on-failure",
|
|
610
|
+
is_flag=True,
|
|
611
|
+
default=True,
|
|
612
|
+
help="Exit with non-zero code if any behavior fails (default: true)",
|
|
613
|
+
)
|
|
614
|
+
@click.option(
|
|
615
|
+
"--output-format",
|
|
616
|
+
default="json",
|
|
617
|
+
type=click.Choice(["json", "summary"]),
|
|
618
|
+
help="Output format (default: json)",
|
|
619
|
+
)
|
|
620
|
+
def ci(agent, behaviors, config, llm_provider, llm_api_key, fail_on_failure, output_format):
|
|
621
|
+
"""
|
|
622
|
+
Run CodeOptiX in CI/CD mode.
|
|
623
|
+
|
|
624
|
+
Optimized for CI/CD pipelines with:
|
|
625
|
+
- Non-interactive execution
|
|
626
|
+
- Exit codes for automation
|
|
627
|
+
- Summary output format
|
|
628
|
+
- Fail-fast behavior
|
|
629
|
+
"""
|
|
630
|
+
import sys
|
|
631
|
+
|
|
632
|
+
click.echo("š CodeOptiX CI/CD Check")
|
|
633
|
+
click.echo("=" * 60)
|
|
634
|
+
|
|
635
|
+
# Parse behaviors
|
|
636
|
+
behavior_list = [b.strip() for b in behaviors.split(",")]
|
|
637
|
+
|
|
638
|
+
if not behavior_list:
|
|
639
|
+
click.echo("ā Error: At least one behavior must be specified", err=True)
|
|
640
|
+
sys.exit(1)
|
|
641
|
+
|
|
642
|
+
# Load config if provided
|
|
643
|
+
config_dict = {}
|
|
644
|
+
if config:
|
|
645
|
+
config_path = Path(config)
|
|
646
|
+
if config_path.suffix == ".json":
|
|
647
|
+
with open(config_path) as f:
|
|
648
|
+
config_dict = json.load(f)
|
|
649
|
+
elif config_path.suffix in [".yaml", ".yml"]:
|
|
650
|
+
import yaml
|
|
651
|
+
|
|
652
|
+
with open(config_path) as f:
|
|
653
|
+
config_dict = yaml.safe_load(f)
|
|
654
|
+
|
|
655
|
+
# Get API key
|
|
656
|
+
api_key = llm_api_key or os.getenv(f"{llm_provider.upper()}_API_KEY")
|
|
657
|
+
if not api_key:
|
|
658
|
+
click.echo(
|
|
659
|
+
f"ā Error: API key required. Set {llm_provider.upper()}_API_KEY environment variable or use --llm-api-key",
|
|
660
|
+
err=True,
|
|
661
|
+
)
|
|
662
|
+
click.echo("", err=True)
|
|
663
|
+
click.echo("š” Tip: Without an API key, you can use basic static analysis:", err=True)
|
|
664
|
+
click.echo(" codeoptix lint --path ./src", err=True)
|
|
665
|
+
click.echo(
|
|
666
|
+
" This runs linters (ruff, bandit, flake8, etc.) without requiring API keys.",
|
|
667
|
+
err=True,
|
|
668
|
+
)
|
|
669
|
+
sys.exit(1)
|
|
670
|
+
|
|
671
|
+
try:
|
|
672
|
+
# Create adapter
|
|
673
|
+
adapter_config = config_dict.get("adapter", {})
|
|
674
|
+
if not adapter_config.get("llm_config"):
|
|
675
|
+
adapter_config["llm_config"] = {
|
|
676
|
+
"provider": llm_provider,
|
|
677
|
+
"api_key": api_key,
|
|
678
|
+
}
|
|
679
|
+
|
|
680
|
+
adapter = create_adapter(agent, adapter_config)
|
|
681
|
+
|
|
682
|
+
# Create LLM client
|
|
683
|
+
llm_provider_enum = LLMProvider[llm_provider.upper()]
|
|
684
|
+
llm_client = create_llm_client(llm_provider_enum, api_key=api_key)
|
|
685
|
+
|
|
686
|
+
# Create evaluation engine
|
|
687
|
+
eval_config = config_dict.get("evaluation", {})
|
|
688
|
+
eval_engine = EvaluationEngine(adapter, llm_client, config=eval_config)
|
|
689
|
+
|
|
690
|
+
# Run evaluation
|
|
691
|
+
click.echo(f"š Evaluating {len(behavior_list)} behavior(s): {', '.join(behavior_list)}")
|
|
692
|
+
|
|
693
|
+
results = eval_engine.evaluate_behaviors(
|
|
694
|
+
behavior_names=behavior_list, context=config_dict.get("context", {})
|
|
695
|
+
)
|
|
696
|
+
|
|
697
|
+
# Save results
|
|
698
|
+
artifact_manager = ArtifactManager()
|
|
699
|
+
run_id = artifact_manager.save_results(results)
|
|
700
|
+
|
|
701
|
+
# Display results
|
|
702
|
+
overall_score = results.get("overall_score", 0.0)
|
|
703
|
+
behaviors_data = results.get("behaviors", {})
|
|
704
|
+
|
|
705
|
+
if output_format == "summary":
|
|
706
|
+
click.echo("\n" + "=" * 60)
|
|
707
|
+
click.echo("š Evaluation Summary")
|
|
708
|
+
click.echo("=" * 60)
|
|
709
|
+
click.echo(f"Overall Score: {overall_score:.2%}")
|
|
710
|
+
click.echo(f"Run ID: {run_id}")
|
|
711
|
+
click.echo()
|
|
712
|
+
|
|
713
|
+
for behavior_name, behavior_data in behaviors_data.items():
|
|
714
|
+
passed = behavior_data.get("passed", True)
|
|
715
|
+
score = behavior_data.get("score", 0.0)
|
|
716
|
+
emoji = "ā
" if passed else "ā"
|
|
717
|
+
click.echo(f"{emoji} {behavior_name}: {score:.2%}")
|
|
718
|
+
|
|
719
|
+
if not passed and behavior_data.get("evidence"):
|
|
720
|
+
evidence = behavior_data["evidence"][:3]
|
|
721
|
+
for ev in evidence:
|
|
722
|
+
click.echo(f" ā ļø {ev}")
|
|
723
|
+
click.echo("=" * 60)
|
|
724
|
+
else:
|
|
725
|
+
# JSON output
|
|
726
|
+
click.echo(
|
|
727
|
+
json.dumps(
|
|
728
|
+
{
|
|
729
|
+
"run_id": run_id,
|
|
730
|
+
"overall_score": overall_score,
|
|
731
|
+
"behaviors": {
|
|
732
|
+
name: {
|
|
733
|
+
"passed": data.get("passed", True),
|
|
734
|
+
"score": data.get("score", 0.0),
|
|
735
|
+
"evidence": data.get("evidence", [])[:3],
|
|
736
|
+
}
|
|
737
|
+
for name, data in behaviors_data.items()
|
|
738
|
+
},
|
|
739
|
+
},
|
|
740
|
+
indent=2,
|
|
741
|
+
)
|
|
742
|
+
)
|
|
743
|
+
|
|
744
|
+
# Check for failures
|
|
745
|
+
failed_behaviors = [
|
|
746
|
+
name for name, data in behaviors_data.items() if not data.get("passed", True)
|
|
747
|
+
]
|
|
748
|
+
|
|
749
|
+
if failed_behaviors:
|
|
750
|
+
if fail_on_failure:
|
|
751
|
+
click.echo(
|
|
752
|
+
f"\nā {len(failed_behaviors)} behavior(s) failed: {', '.join(failed_behaviors)}",
|
|
753
|
+
err=True,
|
|
754
|
+
)
|
|
755
|
+
sys.exit(1)
|
|
756
|
+
else:
|
|
757
|
+
click.echo(
|
|
758
|
+
f"\nā ļø {len(failed_behaviors)} behavior(s) failed: {', '.join(failed_behaviors)}",
|
|
759
|
+
err=True,
|
|
760
|
+
)
|
|
761
|
+
else:
|
|
762
|
+
click.echo("\nā
All behaviors passed!")
|
|
763
|
+
|
|
764
|
+
except Exception as e:
|
|
765
|
+
click.echo(f"ā Error: {e!s}", err=True)
|
|
766
|
+
if hasattr(e, "__cause__") and e.__cause__:
|
|
767
|
+
click.echo(f" Caused by: {e.__cause__!s}", err=True)
|
|
768
|
+
sys.exit(1)
|
|
769
|
+
|
|
770
|
+
|
|
771
|
+
def _get_install_command(linter_name: str) -> str | None:
|
|
772
|
+
"""Get install command for a linter."""
|
|
773
|
+
install_commands = {
|
|
774
|
+
"bandit": "pip install bandit",
|
|
775
|
+
"pylint": "pip install pylint",
|
|
776
|
+
"flake8": "pip install flake8",
|
|
777
|
+
"ruff": "pip install ruff or uv tool install ruff",
|
|
778
|
+
"mypy": "pip install mypy",
|
|
779
|
+
"safety": "pip install safety",
|
|
780
|
+
"pip-audit": "pip install pip-audit",
|
|
781
|
+
"coverage": "pip install coverage",
|
|
782
|
+
"html-accessibility": "No installation needed (built-in)",
|
|
783
|
+
}
|
|
784
|
+
return install_commands.get(linter_name)
|
|
785
|
+
|
|
786
|
+
|
|
787
|
+
@main.command()
|
|
788
|
+
@click.option("--path", type=click.Path(exists=True), help="Path to code (file or directory)")
|
|
789
|
+
@click.option(
|
|
790
|
+
"--linters", help="Comma-separated linter names (default: auto-detect from language and config)"
|
|
791
|
+
)
|
|
792
|
+
@click.option(
|
|
793
|
+
"--output",
|
|
794
|
+
default="summary",
|
|
795
|
+
type=click.Choice(["json", "summary"]),
|
|
796
|
+
help="Output format (default: summary)",
|
|
797
|
+
)
|
|
798
|
+
@click.option("--fail-on-issues", is_flag=True, help="Exit with non-zero code if issues found")
|
|
799
|
+
@click.option(
|
|
800
|
+
"--no-auto-detect", is_flag=True, help="Disable auto-detection of language and linters"
|
|
801
|
+
)
|
|
802
|
+
@click.option("--list-linters", is_flag=True, help="List all available linters and exit")
|
|
803
|
+
def lint(path, linters, output, fail_on_issues, no_auto_detect, list_linters):
|
|
804
|
+
"""
|
|
805
|
+
Run linters on code (no API key required).
|
|
806
|
+
|
|
807
|
+
This command runs static analysis linters on your code without requiring
|
|
808
|
+
any API keys. Perfect for quick code quality checks.
|
|
809
|
+
|
|
810
|
+
Examples:
|
|
811
|
+
codeoptix lint --path ./src
|
|
812
|
+
codeoptix lint --path ./src --linters bandit,flake8
|
|
813
|
+
codeoptix lint --path ./src --output summary
|
|
814
|
+
"""
|
|
815
|
+
import sys
|
|
816
|
+
|
|
817
|
+
# List linters if requested (check this first, before path validation)
|
|
818
|
+
if list_linters:
|
|
819
|
+
runner = LinterRunner()
|
|
820
|
+
available = runner.get_available_linters()
|
|
821
|
+
all_linters = runner.get_all_linters()
|
|
822
|
+
|
|
823
|
+
click.echo("Available Linters (Zero New Dependencies):")
|
|
824
|
+
click.echo("=" * 60)
|
|
825
|
+
click.echo("\nCode Quality:")
|
|
826
|
+
for linter in ["ruff", "pylint", "flake8"]:
|
|
827
|
+
status = "ā
" if linter in available else "ā"
|
|
828
|
+
click.echo(f" {status} {linter}")
|
|
829
|
+
|
|
830
|
+
click.echo("\nType Checking:")
|
|
831
|
+
for linter in ["mypy"]:
|
|
832
|
+
status = "ā
" if linter in available else "ā"
|
|
833
|
+
click.echo(f" {status} {linter}")
|
|
834
|
+
|
|
835
|
+
click.echo("\nSecurity:")
|
|
836
|
+
for linter in ["bandit", "safety", "pip-audit"]:
|
|
837
|
+
status = "ā
" if linter in available else "ā"
|
|
838
|
+
click.echo(f" {status} {linter}")
|
|
839
|
+
|
|
840
|
+
click.echo("\nTesting:")
|
|
841
|
+
for linter in ["coverage"]:
|
|
842
|
+
status = "ā
" if linter in available else "ā"
|
|
843
|
+
click.echo(f" {status} {linter}")
|
|
844
|
+
|
|
845
|
+
click.echo("\nAccessibility:")
|
|
846
|
+
for linter in ["html-accessibility"]:
|
|
847
|
+
status = "ā
" if linter in available else "ā"
|
|
848
|
+
click.echo(f" {status} {linter} (custom, no dependency)")
|
|
849
|
+
|
|
850
|
+
click.echo(f"\nTotal: {len(available)}/{len(all_linters)} linters available")
|
|
851
|
+
click.echo("\nInstall missing linters:")
|
|
852
|
+
for linter in all_linters:
|
|
853
|
+
if linter not in available:
|
|
854
|
+
cmd = _get_install_command(linter)
|
|
855
|
+
if cmd:
|
|
856
|
+
click.echo(f" {cmd}")
|
|
857
|
+
return
|
|
858
|
+
|
|
859
|
+
if not path:
|
|
860
|
+
click.echo("ā Error: --path is required (or use --list-linters)", err=True)
|
|
861
|
+
sys.exit(1)
|
|
862
|
+
|
|
863
|
+
click.echo("š CodeOptiX Linter Check")
|
|
864
|
+
click.echo("=" * 60)
|
|
865
|
+
click.echo(f"š Path: {path}")
|
|
866
|
+
|
|
867
|
+
try:
|
|
868
|
+
# Create linter runner
|
|
869
|
+
runner = LinterRunner()
|
|
870
|
+
|
|
871
|
+
# Auto-detect or use specified linters
|
|
872
|
+
if linters:
|
|
873
|
+
linter_list = [l.strip() for l in linters.split(",") if l.strip()]
|
|
874
|
+
else:
|
|
875
|
+
# Auto-detect from language and existing configs
|
|
876
|
+
# Find Python files if directory
|
|
877
|
+
from pathlib import Path
|
|
878
|
+
|
|
879
|
+
from codeoptix.linters.language_detector import LanguageDetector
|
|
880
|
+
|
|
881
|
+
path_obj = Path(path)
|
|
882
|
+
if path_obj.is_dir():
|
|
883
|
+
python_files = list(path_obj.rglob("*.py"))[:10] # Sample first 10
|
|
884
|
+
file_paths = [str(f) for f in python_files]
|
|
885
|
+
elif path_obj.is_file() and path_obj.suffix == ".py":
|
|
886
|
+
file_paths = [str(path_obj)]
|
|
887
|
+
else:
|
|
888
|
+
file_paths = []
|
|
889
|
+
|
|
890
|
+
# Detect language and find configs
|
|
891
|
+
if file_paths:
|
|
892
|
+
detected_languages = LanguageDetector.detect_languages(file_paths)
|
|
893
|
+
config_files = LanguageDetector.find_config_files(path)
|
|
894
|
+
|
|
895
|
+
click.echo(
|
|
896
|
+
f"š Detected languages: {', '.join(detected_languages) if detected_languages else 'unknown'}"
|
|
897
|
+
)
|
|
898
|
+
|
|
899
|
+
# Get recommended linters
|
|
900
|
+
linter_list = []
|
|
901
|
+
for lang in detected_languages:
|
|
902
|
+
linter_list.extend(LanguageDetector.get_linters_for_language(lang))
|
|
903
|
+
|
|
904
|
+
# Prioritize linters with existing configs
|
|
905
|
+
configured_linters = [l for l in linter_list if l in config_files]
|
|
906
|
+
if configured_linters:
|
|
907
|
+
linter_list = configured_linters + [
|
|
908
|
+
l for l in linter_list if l not in configured_linters
|
|
909
|
+
]
|
|
910
|
+
|
|
911
|
+
# Remove duplicates while preserving order
|
|
912
|
+
seen = set()
|
|
913
|
+
linter_list = [l for l in linter_list if not (l in seen or seen.add(l))]
|
|
914
|
+
|
|
915
|
+
if not linter_list:
|
|
916
|
+
# Fallback to common Python linters
|
|
917
|
+
linter_list = ["ruff", "bandit"] # Ruff first (fastest)
|
|
918
|
+
else:
|
|
919
|
+
linter_list = ["ruff", "bandit"] # Default for non-Python
|
|
920
|
+
|
|
921
|
+
click.echo(f"š§ Linters: {', '.join(linter_list) if linter_list else 'auto-detect'}")
|
|
922
|
+
|
|
923
|
+
# Check available linters
|
|
924
|
+
available = runner.get_available_linters()
|
|
925
|
+
all_linters = runner.get_all_linters()
|
|
926
|
+
requested_available = [l for l in linter_list if l in available]
|
|
927
|
+
|
|
928
|
+
if not requested_available:
|
|
929
|
+
click.echo("ā Error: No requested linters are available", err=True)
|
|
930
|
+
click.echo(f" Available linters: {', '.join(available)}", err=True)
|
|
931
|
+
click.echo(" Install missing linters:", err=True)
|
|
932
|
+
for linter in linter_list:
|
|
933
|
+
if linter not in available:
|
|
934
|
+
install_cmd = _get_install_command(linter)
|
|
935
|
+
if install_cmd:
|
|
936
|
+
click.echo(f" {install_cmd}", err=True)
|
|
937
|
+
sys.exit(1)
|
|
938
|
+
|
|
939
|
+
if len(requested_available) < len(linter_list):
|
|
940
|
+
missing = [l for l in linter_list if l not in available]
|
|
941
|
+
click.echo(f"ā ļø Warning: Some linters not available: {', '.join(missing)}", err=True)
|
|
942
|
+
click.echo(" These linters will be skipped. Install them to use:", err=True)
|
|
943
|
+
for linter in missing:
|
|
944
|
+
install_cmd = _get_install_command(linter)
|
|
945
|
+
if install_cmd:
|
|
946
|
+
click.echo(f" {install_cmd}", err=True)
|
|
947
|
+
|
|
948
|
+
# Run linters
|
|
949
|
+
click.echo("š Running linters...")
|
|
950
|
+
results = runner.run_linters(
|
|
951
|
+
path,
|
|
952
|
+
linter_names=requested_available if requested_available else None,
|
|
953
|
+
auto_detect=not no_auto_detect,
|
|
954
|
+
)
|
|
955
|
+
|
|
956
|
+
# Display results
|
|
957
|
+
summary = results.get("summary", {})
|
|
958
|
+
total_issues = summary.get("total_issues", 0)
|
|
959
|
+
|
|
960
|
+
if output == "summary":
|
|
961
|
+
click.echo("\n" + "=" * 60)
|
|
962
|
+
click.echo("š Linter Results Summary")
|
|
963
|
+
click.echo("=" * 60)
|
|
964
|
+
click.echo(f"Total Issues: {total_issues}")
|
|
965
|
+
click.echo(f" Critical: {summary.get('critical', 0)}")
|
|
966
|
+
click.echo(f" High: {summary.get('high', 0)}")
|
|
967
|
+
click.echo(f" Medium: {summary.get('medium', 0)}")
|
|
968
|
+
click.echo(f" Low: {summary.get('low', 0)}")
|
|
969
|
+
click.echo(f"\nExecution Time: {results.get('execution_time', 0):.2f}s")
|
|
970
|
+
|
|
971
|
+
# Show issues by linter
|
|
972
|
+
linter_results = results.get("results", {})
|
|
973
|
+
for linter_name, linter_result in linter_results.items():
|
|
974
|
+
if isinstance(linter_result, dict):
|
|
975
|
+
issue_count = linter_result.get("issue_count", 0)
|
|
976
|
+
if issue_count > 0:
|
|
977
|
+
click.echo(f"\n{linter_name}: {issue_count} issue(s)")
|
|
978
|
+
|
|
979
|
+
# Show top issues
|
|
980
|
+
issues = results.get("issues", [])
|
|
981
|
+
if issues:
|
|
982
|
+
click.echo("\nā ļø Top Issues:")
|
|
983
|
+
for issue in issues[:10]: # Show top 10
|
|
984
|
+
severity = issue.get("severity", "low").upper()
|
|
985
|
+
file = issue.get("file", "unknown")
|
|
986
|
+
line = issue.get("line", "?")
|
|
987
|
+
message = issue.get("message", "Unknown")
|
|
988
|
+
click.echo(f" [{severity}] {file}:{line} - {message}")
|
|
989
|
+
|
|
990
|
+
click.echo("=" * 60)
|
|
991
|
+
else:
|
|
992
|
+
# JSON output
|
|
993
|
+
click.echo(json.dumps(results, indent=2, default=str))
|
|
994
|
+
|
|
995
|
+
# Check for failures
|
|
996
|
+
if results.get("errors"):
|
|
997
|
+
click.echo("\nā ļø Errors:", err=True)
|
|
998
|
+
for error in results["errors"]:
|
|
999
|
+
click.echo(f" {error}", err=True)
|
|
1000
|
+
|
|
1001
|
+
if total_issues > 0:
|
|
1002
|
+
if fail_on_issues:
|
|
1003
|
+
click.echo(f"\nā Found {total_issues} issue(s)", err=True)
|
|
1004
|
+
sys.exit(1)
|
|
1005
|
+
else:
|
|
1006
|
+
click.echo(f"\nā ļø Found {total_issues} issue(s)")
|
|
1007
|
+
else:
|
|
1008
|
+
click.echo("\nā
No issues found!")
|
|
1009
|
+
|
|
1010
|
+
except Exception as e:
|
|
1011
|
+
click.echo(f"ā Error: {e}", err=True)
|
|
1012
|
+
if hasattr(e, "__cause__") and e.__cause__:
|
|
1013
|
+
click.echo(f" Caused by: {e.__cause__!s}", err=True)
|
|
1014
|
+
sys.exit(1)
|
|
1015
|
+
|
|
1016
|
+
|
|
1017
|
+
@main.command()
|
|
1018
|
+
@click.option("--base", default="main", help="Base branch (default: main)")
|
|
1019
|
+
@click.option("--head", help="Head branch or commit (default: current branch)")
|
|
1020
|
+
@click.option(
|
|
1021
|
+
"--linters", help="Comma-separated linter names (default: auto-detect from language and config)"
|
|
1022
|
+
)
|
|
1023
|
+
@click.option(
|
|
1024
|
+
"--output", default="summary", type=click.Choice(["json", "summary"]), help="Output format"
|
|
1025
|
+
)
|
|
1026
|
+
@click.option("--fail-on-issues", is_flag=True, help="Exit with non-zero code if issues found")
|
|
1027
|
+
@click.option(
|
|
1028
|
+
"--no-auto-detect", is_flag=True, help="Disable auto-detection of language and linters"
|
|
1029
|
+
)
|
|
1030
|
+
def check(base, head, linters, output, fail_on_issues, no_auto_detect):
|
|
1031
|
+
"""
|
|
1032
|
+
Check code changes in git (no API key required).
|
|
1033
|
+
|
|
1034
|
+
This command analyzes code changes between git branches/commits using
|
|
1035
|
+
linters. Perfect for CI/CD pipelines and PR checks.
|
|
1036
|
+
|
|
1037
|
+
Examples:
|
|
1038
|
+
codeoptix check --base main --head feature-branch
|
|
1039
|
+
codeoptix check --base main --head HEAD
|
|
1040
|
+
codeoptix check --linters bandit,flake8
|
|
1041
|
+
"""
|
|
1042
|
+
import subprocess
|
|
1043
|
+
import sys
|
|
1044
|
+
|
|
1045
|
+
click.echo("š CodeOptiX Git Check")
|
|
1046
|
+
click.echo("=" * 60)
|
|
1047
|
+
|
|
1048
|
+
# Get git diff
|
|
1049
|
+
try:
|
|
1050
|
+
# Determine head
|
|
1051
|
+
if not head:
|
|
1052
|
+
result = subprocess.run(
|
|
1053
|
+
["git", "rev-parse", "--abbrev-ref", "HEAD"],
|
|
1054
|
+
capture_output=True,
|
|
1055
|
+
text=True,
|
|
1056
|
+
check=True,
|
|
1057
|
+
)
|
|
1058
|
+
head = result.stdout.strip()
|
|
1059
|
+
|
|
1060
|
+
click.echo(f"š Comparing: {base}..{head}")
|
|
1061
|
+
|
|
1062
|
+
# Get changed files
|
|
1063
|
+
result = subprocess.run(
|
|
1064
|
+
["git", "diff", "--name-only", base, head],
|
|
1065
|
+
capture_output=True,
|
|
1066
|
+
text=True,
|
|
1067
|
+
check=True,
|
|
1068
|
+
)
|
|
1069
|
+
|
|
1070
|
+
changed_files = [f.strip() for f in result.stdout.split("\n") if f.strip()]
|
|
1071
|
+
|
|
1072
|
+
if not changed_files:
|
|
1073
|
+
click.echo("ā
No files changed")
|
|
1074
|
+
return
|
|
1075
|
+
|
|
1076
|
+
# Filter Python files
|
|
1077
|
+
python_files = [f for f in changed_files if f.endswith(".py")]
|
|
1078
|
+
|
|
1079
|
+
if not python_files:
|
|
1080
|
+
click.echo("ā¹ļø No Python files changed")
|
|
1081
|
+
return
|
|
1082
|
+
|
|
1083
|
+
click.echo(f"š Changed Python files: {len(python_files)}")
|
|
1084
|
+
click.echo()
|
|
1085
|
+
|
|
1086
|
+
# Auto-detect or use specified linters
|
|
1087
|
+
from codeoptix.linters.language_detector import LanguageDetector
|
|
1088
|
+
|
|
1089
|
+
if linters:
|
|
1090
|
+
linter_list = [l.strip() for l in linters.split(",") if l.strip()]
|
|
1091
|
+
else:
|
|
1092
|
+
# Auto-detect from changed files
|
|
1093
|
+
detected_languages = LanguageDetector.detect_languages(python_files)
|
|
1094
|
+
config_files = LanguageDetector.find_config_files(".")
|
|
1095
|
+
|
|
1096
|
+
click.echo(
|
|
1097
|
+
f"š Detected languages: {', '.join(detected_languages) if detected_languages else 'Python'}"
|
|
1098
|
+
)
|
|
1099
|
+
|
|
1100
|
+
# Get recommended linters
|
|
1101
|
+
linter_list = []
|
|
1102
|
+
for lang in detected_languages:
|
|
1103
|
+
linter_list.extend(LanguageDetector.get_linters_for_language(lang))
|
|
1104
|
+
|
|
1105
|
+
# Prioritize linters with existing configs
|
|
1106
|
+
configured_linters = [l for l in linter_list if l in config_files]
|
|
1107
|
+
if configured_linters:
|
|
1108
|
+
linter_list = configured_linters + [
|
|
1109
|
+
l for l in linter_list if l not in configured_linters
|
|
1110
|
+
]
|
|
1111
|
+
|
|
1112
|
+
# Remove duplicates
|
|
1113
|
+
seen = set()
|
|
1114
|
+
linter_list = [l for l in linter_list if not (l in seen or seen.add(l))]
|
|
1115
|
+
|
|
1116
|
+
if not linter_list:
|
|
1117
|
+
linter_list = ["ruff", "bandit"] # Default
|
|
1118
|
+
|
|
1119
|
+
click.echo(f"š§ Linters: {', '.join(linter_list)}")
|
|
1120
|
+
|
|
1121
|
+
# Run linters on changed files
|
|
1122
|
+
runner = LinterRunner()
|
|
1123
|
+
|
|
1124
|
+
# Get current directory as base path
|
|
1125
|
+
import os
|
|
1126
|
+
|
|
1127
|
+
base_path = os.getcwd()
|
|
1128
|
+
|
|
1129
|
+
results = runner.run_linters(
|
|
1130
|
+
base_path,
|
|
1131
|
+
linter_names=linter_list,
|
|
1132
|
+
files=python_files,
|
|
1133
|
+
auto_detect=not no_auto_detect,
|
|
1134
|
+
)
|
|
1135
|
+
|
|
1136
|
+
# Filter issues to only changed files
|
|
1137
|
+
all_issues = results.get("issues", [])
|
|
1138
|
+
filtered_issues = [
|
|
1139
|
+
issue
|
|
1140
|
+
for issue in all_issues
|
|
1141
|
+
if any(issue.get("file", "").endswith(f) for f in python_files)
|
|
1142
|
+
]
|
|
1143
|
+
|
|
1144
|
+
# Update summary
|
|
1145
|
+
summary = results.get("summary", {}).copy()
|
|
1146
|
+
summary["total_issues"] = len(filtered_issues)
|
|
1147
|
+
summary["critical"] = sum(1 for i in filtered_issues if i.get("severity") == "critical")
|
|
1148
|
+
summary["high"] = sum(1 for i in filtered_issues if i.get("severity") == "high")
|
|
1149
|
+
summary["medium"] = sum(1 for i in filtered_issues if i.get("severity") == "medium")
|
|
1150
|
+
summary["low"] = sum(1 for i in filtered_issues if i.get("severity") == "low")
|
|
1151
|
+
|
|
1152
|
+
results["summary"] = summary
|
|
1153
|
+
results["issues"] = filtered_issues
|
|
1154
|
+
|
|
1155
|
+
# Display results
|
|
1156
|
+
total_issues = summary.get("total_issues", 0)
|
|
1157
|
+
|
|
1158
|
+
if output == "summary":
|
|
1159
|
+
click.echo("=" * 60)
|
|
1160
|
+
click.echo("š Code Check Results")
|
|
1161
|
+
click.echo("=" * 60)
|
|
1162
|
+
click.echo(f"Total Issues: {total_issues}")
|
|
1163
|
+
click.echo(f" Critical: {summary.get('critical', 0)}")
|
|
1164
|
+
click.echo(f" High: {summary.get('high', 0)}")
|
|
1165
|
+
click.echo(f" Medium: {summary.get('medium', 0)}")
|
|
1166
|
+
click.echo(f" Low: {summary.get('low', 0)}")
|
|
1167
|
+
|
|
1168
|
+
if filtered_issues:
|
|
1169
|
+
click.echo("\nā ļø Issues in Changed Files:")
|
|
1170
|
+
for issue in filtered_issues[:20]: # Show top 20
|
|
1171
|
+
severity = issue.get("severity", "low").upper()
|
|
1172
|
+
file = issue.get("file", "unknown")
|
|
1173
|
+
line = issue.get("line", "?")
|
|
1174
|
+
message = issue.get("message", "Unknown")
|
|
1175
|
+
click.echo(f" [{severity}] {file}:{line} - {message}")
|
|
1176
|
+
|
|
1177
|
+
click.echo("=" * 60)
|
|
1178
|
+
else:
|
|
1179
|
+
click.echo(json.dumps(results, indent=2, default=str))
|
|
1180
|
+
|
|
1181
|
+
if total_issues > 0:
|
|
1182
|
+
if fail_on_issues:
|
|
1183
|
+
click.echo(f"\nā Found {total_issues} issue(s) in changed files", err=True)
|
|
1184
|
+
sys.exit(1)
|
|
1185
|
+
else:
|
|
1186
|
+
click.echo(f"\nā ļø Found {total_issues} issue(s) in changed files")
|
|
1187
|
+
else:
|
|
1188
|
+
click.echo("\nā
No issues found in changed files!")
|
|
1189
|
+
|
|
1190
|
+
except subprocess.CalledProcessError as e:
|
|
1191
|
+
click.echo(f"ā Git error: {e.stderr}", err=True)
|
|
1192
|
+
sys.exit(1)
|
|
1193
|
+
except FileNotFoundError:
|
|
1194
|
+
click.echo("ā Error: Git not found. Please install git.", err=True)
|
|
1195
|
+
sys.exit(1)
|
|
1196
|
+
except Exception as e:
|
|
1197
|
+
click.echo(f"ā Error: {e}", err=True)
|
|
1198
|
+
sys.exit(1)
|
|
1199
|
+
|
|
1200
|
+
|
|
1201
|
+
@main.command()
|
|
1202
|
+
def list_runs():
|
|
1203
|
+
"""List all evaluation runs."""
|
|
1204
|
+
artifact_manager = ArtifactManager()
|
|
1205
|
+
|
|
1206
|
+
runs = artifact_manager.list_runs()
|
|
1207
|
+
|
|
1208
|
+
if not runs:
|
|
1209
|
+
click.echo("No evaluation runs found.")
|
|
1210
|
+
return
|
|
1211
|
+
|
|
1212
|
+
click.echo(f"Found {len(runs)} evaluation run(s):\n")
|
|
1213
|
+
|
|
1214
|
+
for run in runs:
|
|
1215
|
+
click.echo(f"Run ID: {run['run_id']}")
|
|
1216
|
+
click.echo(f" Timestamp: {run.get('timestamp', 'unknown')}")
|
|
1217
|
+
click.echo(f" Score: {run.get('overall_score', 0.0):.2f}/1.0")
|
|
1218
|
+
click.echo(f" Behaviors: {', '.join(run.get('behaviors', []))}")
|
|
1219
|
+
click.echo()
|
|
1220
|
+
|
|
1221
|
+
|
|
1222
|
+
@main.group()
|
|
1223
|
+
def acp():
|
|
1224
|
+
"""ACP (Agent Client Protocol) integration commands."""
|
|
1225
|
+
|
|
1226
|
+
|
|
1227
|
+
@acp.command()
|
|
1228
|
+
def register():
|
|
1229
|
+
"""Register CodeOptiX as an ACP agent (for use with editors like Zed, JetBrains, Neovim)."""
|
|
1230
|
+
click.echo("š Starting CodeOptiX as ACP agent...")
|
|
1231
|
+
click.echo("š CodeOptiX will be available to ACP-compatible editors")
|
|
1232
|
+
click.echo("š” Connect from your editor using ACP protocol")
|
|
1233
|
+
click.echo()
|
|
1234
|
+
|
|
1235
|
+
# Create CodeOptiX agent
|
|
1236
|
+
agent = CodeOptiXAgent()
|
|
1237
|
+
|
|
1238
|
+
# Run agent (this blocks and handles ACP protocol)
|
|
1239
|
+
try:
|
|
1240
|
+
asyncio.run(run_agent(agent))
|
|
1241
|
+
except KeyboardInterrupt:
|
|
1242
|
+
click.echo("\nš CodeOptiX ACP agent stopped")
|
|
1243
|
+
|
|
1244
|
+
|
|
1245
|
+
@acp.command()
|
|
1246
|
+
@click.option("--agent-command", help="Command to spawn ACP agent (e.g., 'python agent.py')")
|
|
1247
|
+
@click.option("--agent-name", help="Name of agent in registry (alternative to agent-command)")
|
|
1248
|
+
@click.option(
|
|
1249
|
+
"--auto-eval/--no-auto-eval", default=True, help="Automatically evaluate code quality"
|
|
1250
|
+
)
|
|
1251
|
+
@click.option("--cwd", help="Working directory for the agent")
|
|
1252
|
+
@click.option("--behaviors", help="Comma-separated behavior names to evaluate")
|
|
1253
|
+
def bridge(
|
|
1254
|
+
agent_command: str | None,
|
|
1255
|
+
agent_name: str | None,
|
|
1256
|
+
auto_eval: bool,
|
|
1257
|
+
cwd: str | None,
|
|
1258
|
+
behaviors: str | None,
|
|
1259
|
+
):
|
|
1260
|
+
"""Use CodeOptiX as a quality bridge between editor and agent via ACP."""
|
|
1261
|
+
if not agent_command and not agent_name:
|
|
1262
|
+
click.echo("ā Error: Either --agent-command or --agent-name must be provided", err=True)
|
|
1263
|
+
raise click.Abort()
|
|
1264
|
+
|
|
1265
|
+
click.echo("š Starting CodeOptiX ACP Quality Bridge...")
|
|
1266
|
+
if agent_command:
|
|
1267
|
+
click.echo(f"š¤ Agent command: {agent_command}")
|
|
1268
|
+
if agent_name:
|
|
1269
|
+
click.echo(f"š¤ Agent name: {agent_name}")
|
|
1270
|
+
click.echo(f"š Auto-evaluation: {auto_eval}")
|
|
1271
|
+
click.echo()
|
|
1272
|
+
|
|
1273
|
+
# Parse behaviors
|
|
1274
|
+
behavior_list = behaviors.split(",") if behaviors else None
|
|
1275
|
+
|
|
1276
|
+
# Create evaluation engine if auto_eval
|
|
1277
|
+
evaluation_engine = None
|
|
1278
|
+
llm_client = None
|
|
1279
|
+
if auto_eval:
|
|
1280
|
+
from codeoptix.adapters.factory import create_adapter
|
|
1281
|
+
from codeoptix.evaluation import EvaluationEngine
|
|
1282
|
+
from codeoptix.utils.llm import LLMProvider, create_llm_client
|
|
1283
|
+
|
|
1284
|
+
# Create a dummy adapter for evaluation
|
|
1285
|
+
adapter = create_adapter("claude-code", {})
|
|
1286
|
+
llm_client = create_llm_client(LLMProvider.OPENAI)
|
|
1287
|
+
evaluation_engine = EvaluationEngine(adapter, llm_client)
|
|
1288
|
+
|
|
1289
|
+
# Create registry if using agent_name
|
|
1290
|
+
registry = None
|
|
1291
|
+
if agent_name:
|
|
1292
|
+
registry = ACPAgentRegistry()
|
|
1293
|
+
# Agent should be pre-registered, but we'll handle it
|
|
1294
|
+
|
|
1295
|
+
# Parse agent command if provided
|
|
1296
|
+
agent_cmd = agent_command.split() if agent_command else None
|
|
1297
|
+
|
|
1298
|
+
# Create quality bridge
|
|
1299
|
+
bridge = ACPQualityBridge(
|
|
1300
|
+
agent_command=agent_cmd,
|
|
1301
|
+
agent_name=agent_name,
|
|
1302
|
+
evaluation_engine=evaluation_engine,
|
|
1303
|
+
llm_client=llm_client,
|
|
1304
|
+
auto_eval=auto_eval,
|
|
1305
|
+
registry=registry,
|
|
1306
|
+
behaviors=behavior_list,
|
|
1307
|
+
)
|
|
1308
|
+
|
|
1309
|
+
async def run_bridge():
|
|
1310
|
+
await bridge.connect(cwd=cwd)
|
|
1311
|
+
click.echo("ā
Quality bridge connected!")
|
|
1312
|
+
click.echo("š” CodeOptiX will now evaluate all agent interactions")
|
|
1313
|
+
# Keep bridge running
|
|
1314
|
+
try:
|
|
1315
|
+
await asyncio.Event().wait()
|
|
1316
|
+
except KeyboardInterrupt:
|
|
1317
|
+
click.echo("\nš Quality bridge stopped")
|
|
1318
|
+
await bridge.close()
|
|
1319
|
+
|
|
1320
|
+
try:
|
|
1321
|
+
asyncio.run(run_bridge())
|
|
1322
|
+
except KeyboardInterrupt:
|
|
1323
|
+
click.echo("\nš CodeOptiX quality bridge stopped")
|
|
1324
|
+
|
|
1325
|
+
|
|
1326
|
+
@acp.command()
|
|
1327
|
+
@click.option("--agent-command", help="Command to spawn ACP agent")
|
|
1328
|
+
@click.option("--agent-name", help="Name of agent in registry")
|
|
1329
|
+
@click.option("--prompt", required=True, help="Prompt to send to agent")
|
|
1330
|
+
@click.option("--cwd", help="Working directory")
|
|
1331
|
+
def connect(agent_command: str | None, agent_name: str | None, prompt: str, cwd: str | None):
|
|
1332
|
+
"""Connect to an ACP agent and send a prompt."""
|
|
1333
|
+
if not agent_command and not agent_name:
|
|
1334
|
+
click.echo("ā Error: Either --agent-command or --agent-name must be provided", err=True)
|
|
1335
|
+
raise click.Abort()
|
|
1336
|
+
|
|
1337
|
+
if agent_command:
|
|
1338
|
+
click.echo(f"š Connecting to ACP agent: {agent_command}")
|
|
1339
|
+
if agent_name:
|
|
1340
|
+
click.echo(f"š Connecting to ACP agent: {agent_name}")
|
|
1341
|
+
|
|
1342
|
+
# Parse agent command if provided
|
|
1343
|
+
agent_cmd = agent_command.split() if agent_command else None
|
|
1344
|
+
|
|
1345
|
+
# Create registry if using agent_name
|
|
1346
|
+
registry = None
|
|
1347
|
+
if agent_name:
|
|
1348
|
+
registry = ACPAgentRegistry()
|
|
1349
|
+
|
|
1350
|
+
# Create bridge and send prompt
|
|
1351
|
+
bridge = ACPQualityBridge(
|
|
1352
|
+
agent_command=agent_cmd,
|
|
1353
|
+
agent_name=agent_name,
|
|
1354
|
+
auto_eval=True,
|
|
1355
|
+
registry=registry,
|
|
1356
|
+
)
|
|
1357
|
+
|
|
1358
|
+
async def run_connect():
|
|
1359
|
+
await bridge.connect(cwd=cwd)
|
|
1360
|
+
click.echo("ā
Connected!")
|
|
1361
|
+
click.echo(f"š¤ Sending prompt: {prompt[:50]}...")
|
|
1362
|
+
result = await bridge.prompt(prompt)
|
|
1363
|
+
click.echo(f"ā
Response: {result}")
|
|
1364
|
+
await bridge.close()
|
|
1365
|
+
|
|
1366
|
+
try:
|
|
1367
|
+
asyncio.run(run_connect())
|
|
1368
|
+
except Exception as e:
|
|
1369
|
+
click.echo(f"ā Error: {e}", err=True)
|
|
1370
|
+
raise click.Abort()
|
|
1371
|
+
|
|
1372
|
+
|
|
1373
|
+
@acp.group()
|
|
1374
|
+
def registry():
|
|
1375
|
+
"""Manage ACP agent registry."""
|
|
1376
|
+
|
|
1377
|
+
|
|
1378
|
+
@registry.command("list")
|
|
1379
|
+
def registry_list():
|
|
1380
|
+
"""List all registered ACP agents."""
|
|
1381
|
+
registry = ACPAgentRegistry()
|
|
1382
|
+
agents = registry.list_agents()
|
|
1383
|
+
|
|
1384
|
+
if not agents:
|
|
1385
|
+
click.echo("No agents registered.")
|
|
1386
|
+
return
|
|
1387
|
+
|
|
1388
|
+
click.echo(f"Registered ACP agents ({len(agents)}):\n")
|
|
1389
|
+
for agent_name in agents:
|
|
1390
|
+
config = registry.get_agent(agent_name)
|
|
1391
|
+
click.echo(f" ⢠{agent_name}")
|
|
1392
|
+
if config and config.description:
|
|
1393
|
+
click.echo(f" {config.description}")
|
|
1394
|
+
if config and config.command:
|
|
1395
|
+
click.echo(f" Command: {' '.join(config.command)}")
|
|
1396
|
+
|
|
1397
|
+
|
|
1398
|
+
@registry.command("add")
|
|
1399
|
+
@click.option("--name", required=True, help="Agent name")
|
|
1400
|
+
@click.option("--command", required=True, help="Command to spawn agent (e.g., 'python agent.py')")
|
|
1401
|
+
@click.option("--cwd", help="Working directory")
|
|
1402
|
+
@click.option("--description", help="Agent description")
|
|
1403
|
+
def registry_add(name: str, command: str, cwd: str | None, description: str | None):
|
|
1404
|
+
"""Register a new ACP agent."""
|
|
1405
|
+
registry = ACPAgentRegistry()
|
|
1406
|
+
registry.register(
|
|
1407
|
+
name=name,
|
|
1408
|
+
command=command.split(),
|
|
1409
|
+
cwd=cwd,
|
|
1410
|
+
description=description or "",
|
|
1411
|
+
)
|
|
1412
|
+
click.echo(f"ā
Registered agent: {name}")
|
|
1413
|
+
|
|
1414
|
+
|
|
1415
|
+
@registry.command("remove")
|
|
1416
|
+
@click.option("--name", required=True, help="Agent name")
|
|
1417
|
+
def registry_remove(name: str):
|
|
1418
|
+
"""Unregister an ACP agent."""
|
|
1419
|
+
registry = ACPAgentRegistry()
|
|
1420
|
+
registry.unregister(name)
|
|
1421
|
+
click.echo(f"ā
Unregistered agent: {name}")
|
|
1422
|
+
|
|
1423
|
+
|
|
1424
|
+
@acp.command()
|
|
1425
|
+
@click.option("--generate-agent", required=True, help="Agent name for code generation")
|
|
1426
|
+
@click.option("--judge-agent", required=True, help="Agent name for code judgment")
|
|
1427
|
+
@click.option("--prompt", required=True, help="Prompt for code generation")
|
|
1428
|
+
def judge(generate_agent: str, judge_agent: str, prompt: str):
|
|
1429
|
+
"""Use multi-agent judge: generate with one agent, judge with another."""
|
|
1430
|
+
click.echo("āļø Starting Multi-Agent Judge...")
|
|
1431
|
+
click.echo(f"š¤ Generate agent: {generate_agent}")
|
|
1432
|
+
click.echo(f"āļø Judge agent: {judge_agent}")
|
|
1433
|
+
click.echo()
|
|
1434
|
+
|
|
1435
|
+
# Create registry
|
|
1436
|
+
registry = ACPAgentRegistry()
|
|
1437
|
+
|
|
1438
|
+
# Create evaluation engine
|
|
1439
|
+
from codeoptix.adapters.factory import create_adapter
|
|
1440
|
+
from codeoptix.evaluation import EvaluationEngine
|
|
1441
|
+
from codeoptix.utils.llm import LLMProvider, create_llm_client
|
|
1442
|
+
|
|
1443
|
+
adapter = create_adapter("claude-code", {})
|
|
1444
|
+
llm_client = create_llm_client(LLMProvider.OPENAI)
|
|
1445
|
+
evaluation_engine = EvaluationEngine(adapter, llm_client)
|
|
1446
|
+
|
|
1447
|
+
# Create multi-agent judge
|
|
1448
|
+
judge = MultiAgentJudge(
|
|
1449
|
+
registry=registry,
|
|
1450
|
+
generate_agent=generate_agent,
|
|
1451
|
+
judge_agent=judge_agent,
|
|
1452
|
+
evaluation_engine=evaluation_engine,
|
|
1453
|
+
llm_client=llm_client,
|
|
1454
|
+
)
|
|
1455
|
+
|
|
1456
|
+
async def run_judge():
|
|
1457
|
+
result = await judge.generate_and_judge(prompt)
|
|
1458
|
+
click.echo("ā
Multi-agent judge complete!")
|
|
1459
|
+
click.echo(f"\nš Generated Code:\n{result.get('generated_code', 'N/A')}")
|
|
1460
|
+
click.echo(f"\nāļø Judgment:\n{result.get('judgment', 'N/A')}")
|
|
1461
|
+
if result.get("evaluation_results"):
|
|
1462
|
+
click.echo(f"\nš Evaluation Results:\n{result['evaluation_results']}")
|
|
1463
|
+
|
|
1464
|
+
try:
|
|
1465
|
+
asyncio.run(run_judge())
|
|
1466
|
+
except Exception as e:
|
|
1467
|
+
click.echo(f"ā Error: {e}", err=True)
|
|
1468
|
+
raise click.Abort()
|
|
1469
|
+
|
|
1470
|
+
|
|
1471
|
+
if __name__ == "__main__":
|
|
1472
|
+
main()
|