omega-code 0.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- omega/__init__.py +0 -0
- omega/__main__.py +589 -0
- omega/artifacts.py +151 -0
- omega/checkpoint.py +246 -0
- omega/compact.py +106 -0
- omega/config.py +285 -0
- omega/eval/__init__.py +3 -0
- omega/eval/cli.py +127 -0
- omega/eval/examples/plan-version-flag.yaml +11 -0
- omega/eval/examples/relative-age-negative-delta.yaml +14 -0
- omega/eval/examples/version-flag.yaml +10 -0
- omega/eval/manifest.py +129 -0
- omega/eval/prices.py +29 -0
- omega/eval/report.py +135 -0
- omega/eval/runner.py +199 -0
- omega/eval/tasks.py +97 -0
- omega/events.py +145 -0
- omega/export.py +80 -0
- omega/gitlog.py +229 -0
- omega/hooks.py +63 -0
- omega/instructions.py +103 -0
- omega/integrations.py +284 -0
- omega/keys.py +173 -0
- omega/llm.py +442 -0
- omega/loop.py +510 -0
- omega/mcp.py +490 -0
- omega/memory/__init__.py +5 -0
- omega/memory/consolidate.py +103 -0
- omega/memory/curate.py +69 -0
- omega/memory/store.py +321 -0
- omega/memory/tools.py +175 -0
- omega/migrate.py +40 -0
- omega/onboarding.py +242 -0
- omega/permissions.py +137 -0
- omega/secrets.py +173 -0
- omega/server/__init__.py +7 -0
- omega/server/__main__.py +18 -0
- omega/server/app.py +71 -0
- omega/server/auth.py +73 -0
- omega/server/manager.py +287 -0
- omega/server/models.py +123 -0
- omega/server/tasks_api.py +311 -0
- omega/server/terminals.py +245 -0
- omega/server/worker.py +186 -0
- omega/session.py +209 -0
- omega/setup.html +281 -0
- omega/setup_server.py +452 -0
- omega/skills.py +158 -0
- omega/subagent.py +98 -0
- omega/tasks.py +195 -0
- omega/tools.py +590 -0
- omega/trace.py +156 -0
- omega/trajectory.py +146 -0
- omega/ui/__init__.py +0 -0
- omega/ui/composer.py +140 -0
- omega/ui/format.py +708 -0
- omega/ui/plain.py +141 -0
- omega/ui/tui/__init__.py +9 -0
- omega/ui/tui/app.py +958 -0
- omega/ui/tui/history.py +50 -0
- omega/ui/tui/modals.py +292 -0
- omega/ui/tui/onboarding.py +367 -0
- omega/ui/tui/prefs.py +25 -0
- omega/ui/tui/sidebar.py +510 -0
- omega/ui/tui/status.py +115 -0
- omega/ui/tui/theme.py +91 -0
- omega/ui/tui/transcript.py +783 -0
- omega/verify.py +133 -0
- omega_code-0.4.0.dist-info/METADATA +479 -0
- omega_code-0.4.0.dist-info/RECORD +73 -0
- omega_code-0.4.0.dist-info/WHEEL +4 -0
- omega_code-0.4.0.dist-info/entry_points.txt +2 -0
- omega_code-0.4.0.dist-info/licenses/LICENSE +21 -0
omega/config.py
ADDED
|
@@ -0,0 +1,285 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import os
|
|
3
|
+
import re
|
|
4
|
+
from dataclasses import dataclass, field
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import Any, Literal
|
|
7
|
+
|
|
8
|
+
from . import secrets
|
|
9
|
+
|
|
10
|
+
CONFIG_PATH = Path(os.environ.get("OMEGA_CONFIG", Path.home() / ".omega" / "config.json"))
|
|
11
|
+
|
|
12
|
+
DEFAULTS: dict[str, Any] = {
|
|
13
|
+
"providers": {
|
|
14
|
+
"inference-net": {
|
|
15
|
+
"baseUrl": "https://api.inference.net/v1",
|
|
16
|
+
"apiKeyEnv": "INFERENCE_API_KEY",
|
|
17
|
+
},
|
|
18
|
+
"openrouter": {
|
|
19
|
+
"baseUrl": "https://openrouter.ai/api/v1",
|
|
20
|
+
"apiKeyEnv": "OPENROUTER_API_KEY",
|
|
21
|
+
},
|
|
22
|
+
"anthropic": {
|
|
23
|
+
"type": "anthropic",
|
|
24
|
+
"apiKeyEnv": "ANTHROPIC_API_KEY",
|
|
25
|
+
},
|
|
26
|
+
"openai": {
|
|
27
|
+
"baseUrl": "https://api.openai.com/v1",
|
|
28
|
+
"apiKeyEnv": "OPENAI_API_KEY",
|
|
29
|
+
},
|
|
30
|
+
},
|
|
31
|
+
"models": {
|
|
32
|
+
"fable": {"model": "claude-fable-5-1", "provider": "anthropic", "context": 1048576,
|
|
33
|
+
"effort": "xhigh", "fallback": "opus"},
|
|
34
|
+
"opus": {"model": "claude-opus-5", "provider": "anthropic", "context": 1048576,
|
|
35
|
+
"effort": "high", "fallback": "sonnet"},
|
|
36
|
+
"sonnet": {"model": "claude-sonnet-5", "provider": "anthropic", "context": 1048576,
|
|
37
|
+
"effort": "high", "fallback": "haiku"},
|
|
38
|
+
"haiku": {"model": "claude-haiku-4-5", "provider": "anthropic", "context": 200000},
|
|
39
|
+
"spark": {"model": "meta/muse-spark-1.3", "provider": "openrouter", "context": 1048576,
|
|
40
|
+
"fallback": "kimi"},
|
|
41
|
+
"kimi": {"model": "moonshotai/kimi-k3", "provider": "openrouter", "context": 1048576},
|
|
42
|
+
"glm": {"model": "z-ai/glm-5.3-flash", "provider": "openrouter", "context": 128000},
|
|
43
|
+
# GPT-6 Astra is in limited rollout and not on OpenRouter yet, so it
|
|
44
|
+
# needs the native OpenAI provider; the GPT-5.6 tiers are on both.
|
|
45
|
+
"astra": {"model": "gpt-6-astra", "provider": "openai", "context": 1050000,
|
|
46
|
+
"fallback": "sol"},
|
|
47
|
+
"sol": {"model": "openai/gpt-5.6-sol", "provider": "openrouter", "context": 1050000,
|
|
48
|
+
"fallback": "terra"},
|
|
49
|
+
"terra": {"model": "openai/gpt-5.6-terra", "provider": "openrouter", "context": 1050000,
|
|
50
|
+
"fallback": "luna"},
|
|
51
|
+
"luna": {"model": "openai/gpt-5.6-luna", "provider": "openrouter", "context": 1050000},
|
|
52
|
+
"codex": {"model": "openai/gpt-5.3-codex", "provider": "openrouter", "context": 400000,
|
|
53
|
+
"fallback": "sol"},
|
|
54
|
+
"grok": {"model": "x-ai/grok-4.6", "provider": "openrouter", "context": 500000,
|
|
55
|
+
"fallback": "grok-build"},
|
|
56
|
+
"grok-build": {"model": "x-ai/grok-build-0.1", "provider": "openrouter", "context": 256000},
|
|
57
|
+
},
|
|
58
|
+
"roles": {
|
|
59
|
+
"main": {"alias": "opus"},
|
|
60
|
+
"plan": {"alias": "opus"},
|
|
61
|
+
"subagent_fast": {"alias": "glm"},
|
|
62
|
+
"subagent_mid": {"alias": "kimi"},
|
|
63
|
+
"compact": {"alias": "glm"},
|
|
64
|
+
"memory": {"alias": "glm"},
|
|
65
|
+
},
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
@dataclass
|
|
70
|
+
class Provider:
|
|
71
|
+
name: str
|
|
72
|
+
type: Literal["openai", "anthropic"] = "openai"
|
|
73
|
+
base_url: str = ""
|
|
74
|
+
api_key_env: str = ""
|
|
75
|
+
api_key_literal: str = ""
|
|
76
|
+
api_key_cmd: str = ""
|
|
77
|
+
|
|
78
|
+
def _resolve(self) -> str:
|
|
79
|
+
"""The key, or "" if nothing supplies one.
|
|
80
|
+
|
|
81
|
+
Order is most-explicit-first: a literal in the config, then a command,
|
|
82
|
+
then an environment variable, then the OS keychain. The keychain is
|
|
83
|
+
last because it is the one source the config does not mention -- so a
|
|
84
|
+
key written down in front of you always beats one found by
|
|
85
|
+
convention, and `omega keys migrate` can move a literal into the
|
|
86
|
+
keychain without the two fighting over which wins."""
|
|
87
|
+
if self.api_key_literal:
|
|
88
|
+
return self.api_key_literal
|
|
89
|
+
if self.api_key_cmd:
|
|
90
|
+
return secrets.from_command(self.api_key_cmd)
|
|
91
|
+
if self.api_key_env:
|
|
92
|
+
from_env = os.environ.get(self.api_key_env, "")
|
|
93
|
+
if from_env:
|
|
94
|
+
return from_env
|
|
95
|
+
return secrets.keychain_get(self.name) or ""
|
|
96
|
+
|
|
97
|
+
@property
|
|
98
|
+
def has_key(self) -> bool:
|
|
99
|
+
"""Non-raising check -- lets callers (onboarding, first-run detection)
|
|
100
|
+
probe key availability without triggering the SystemExit below."""
|
|
101
|
+
try:
|
|
102
|
+
return bool(self._resolve())
|
|
103
|
+
except RuntimeError:
|
|
104
|
+
# A broken `apiKeyCmd` is a real misconfiguration, but this
|
|
105
|
+
# property exists precisely so probing cannot blow up its caller.
|
|
106
|
+
return False
|
|
107
|
+
|
|
108
|
+
@property
|
|
109
|
+
def key_source(self) -> str:
|
|
110
|
+
"""Where the key comes from, for `omega keys` and `omega doctor`.
|
|
111
|
+
Never the value itself."""
|
|
112
|
+
return secrets.describe_source(self.api_key_env, self.api_key_cmd,
|
|
113
|
+
self.api_key_literal, self.name)
|
|
114
|
+
|
|
115
|
+
@property
|
|
116
|
+
def api_key(self) -> str:
|
|
117
|
+
# Resolved lazily -- at load() time we don't yet know which providers a
|
|
118
|
+
# session will actually use, and a provider with no key configured must
|
|
119
|
+
# not block startup for users who haven't set it up yet.
|
|
120
|
+
try:
|
|
121
|
+
key = self._resolve()
|
|
122
|
+
except RuntimeError as e:
|
|
123
|
+
raise SystemExit(f"omega: could not read the API key for provider "
|
|
124
|
+
f"{self.name!r}: {secrets.redact(str(e))}") from None
|
|
125
|
+
if not key:
|
|
126
|
+
hint = (f"export {self.api_key_env}=..." if self.api_key_env
|
|
127
|
+
else f"omega keys set {self.name}")
|
|
128
|
+
raise SystemExit(
|
|
129
|
+
f"omega: no API key for provider {self.name!r}.\n"
|
|
130
|
+
f" Run `omega setup` to configure one, or {hint}")
|
|
131
|
+
return key
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
@dataclass
|
|
135
|
+
class Model:
|
|
136
|
+
alias: str
|
|
137
|
+
model: str
|
|
138
|
+
provider: str
|
|
139
|
+
context: int = 128000
|
|
140
|
+
effort: str | None = None
|
|
141
|
+
fallback: str | None = None
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
@dataclass
|
|
145
|
+
class Role:
|
|
146
|
+
model: str
|
|
147
|
+
provider: Provider
|
|
148
|
+
context: int = 128000
|
|
149
|
+
effort: str | None = None
|
|
150
|
+
alias: str | None = None
|
|
151
|
+
fallback_alias: str | None = None
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
@dataclass(frozen=True)
|
|
155
|
+
class HookRule:
|
|
156
|
+
"""One `hooks.json` entry: `command` runs (with OMEGA_* env vars) whenever a
|
|
157
|
+
call to one of `tools` is dispatched -- see hooks.py."""
|
|
158
|
+
tools: list[str]
|
|
159
|
+
command: str
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
@dataclass
|
|
163
|
+
class Config:
|
|
164
|
+
roles: dict[str, Role] = field(default_factory=dict)
|
|
165
|
+
models: dict[str, Model] = field(default_factory=dict)
|
|
166
|
+
providers: dict[str, Provider] = field(default_factory=dict)
|
|
167
|
+
# B1 edit-safety config: read from top-level "verify"/"hooks"/"review_auto"
|
|
168
|
+
# keys in config.json -- see verify.py, hooks.py and subagent.review().
|
|
169
|
+
verify_auto: bool = True
|
|
170
|
+
verify_checks: list[str] | None = None
|
|
171
|
+
review_auto: bool = True
|
|
172
|
+
hooks: dict[str, list[HookRule]] = field(default_factory=dict)
|
|
173
|
+
|
|
174
|
+
def role(self, name: str) -> Role:
|
|
175
|
+
if name not in self.roles:
|
|
176
|
+
raise KeyError(f"no role {name!r}; have {sorted(self.roles)}")
|
|
177
|
+
return self.roles[name]
|
|
178
|
+
|
|
179
|
+
def model(self, alias: str) -> Role:
|
|
180
|
+
if alias not in self.models:
|
|
181
|
+
raise KeyError(f"no model {alias!r}; have {sorted(self.models)}")
|
|
182
|
+
m = self.models[alias]
|
|
183
|
+
if m.provider not in self.providers:
|
|
184
|
+
raise KeyError(f"model {alias!r} references unknown provider {m.provider!r}")
|
|
185
|
+
return Role(m.model, self.providers[m.provider], m.context, m.effort, alias, m.fallback)
|
|
186
|
+
|
|
187
|
+
def resolve_alias(self, text: str) -> str:
|
|
188
|
+
"""Resolve a `--model`/`/model` argument to a catalog alias: an exact
|
|
189
|
+
alias match first, else a bare model id matched against catalog entries."""
|
|
190
|
+
if text in self.models:
|
|
191
|
+
return text
|
|
192
|
+
for alias, m in self.models.items():
|
|
193
|
+
if m.model == text:
|
|
194
|
+
return alias
|
|
195
|
+
raise SystemExit(f"omega: unknown model {text!r}; have {sorted(self.models)}")
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def _strip_jsonc(text: str) -> str:
|
|
199
|
+
text = re.sub(r"^\s*//.*$", "", text, flags=re.M)
|
|
200
|
+
return re.sub(r",(\s*[}\]])", r"\1", text)
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def _json_or_default() -> dict[str, Any]:
|
|
204
|
+
"""The raw config dict as written on disk, or an empty skeleton -- used by
|
|
205
|
+
onboarding to merge its additions into whatever is already there instead
|
|
206
|
+
of clobbering unrelated providers/roles/mcp entries."""
|
|
207
|
+
if CONFIG_PATH.exists():
|
|
208
|
+
return dict(json.loads(_strip_jsonc(CONFIG_PATH.read_text())))
|
|
209
|
+
return {"providers": {}, "models": {}, "roles": {}}
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
def load() -> Config:
|
|
213
|
+
raw: dict[str, Any] = DEFAULTS
|
|
214
|
+
if CONFIG_PATH.exists():
|
|
215
|
+
raw = json.loads(_strip_jsonc(CONFIG_PATH.read_text()))
|
|
216
|
+
|
|
217
|
+
providers: dict[str, Provider] = {}
|
|
218
|
+
for name, p in raw["providers"].items():
|
|
219
|
+
ptype = p.get("type", "openai")
|
|
220
|
+
# Only strip a trailing slash -- an anthropic provider's baseUrl (when
|
|
221
|
+
# given at all) is passed straight to AsyncAnthropic as base_url=, and
|
|
222
|
+
# the SDK's own default already omits a "/v1" suffix.
|
|
223
|
+
base_url = (p.get("baseUrl") or "").rstrip("/")
|
|
224
|
+
if ptype == "openai" and not base_url:
|
|
225
|
+
raise SystemExit(f"omega: provider {name!r} is missing \"baseUrl\" in {CONFIG_PATH}")
|
|
226
|
+
providers[name] = Provider(
|
|
227
|
+
name=name, type=ptype, base_url=base_url,
|
|
228
|
+
api_key_env=p.get("apiKeyEnv", ""),
|
|
229
|
+
api_key_literal=p.get("apiKey", ""),
|
|
230
|
+
api_key_cmd=p.get("apiKeyCmd", ""),
|
|
231
|
+
)
|
|
232
|
+
|
|
233
|
+
models: dict[str, Model] = {}
|
|
234
|
+
for alias, m in (raw.get("models") or {}).items():
|
|
235
|
+
models[alias] = Model(alias, m["model"], m["provider"], m.get("context", 128000),
|
|
236
|
+
m.get("effort"), m.get("fallback"))
|
|
237
|
+
# A hand-written config predating the catalog would otherwise leave the
|
|
238
|
+
# /model picker empty; built-ins fill in wherever their provider exists.
|
|
239
|
+
for alias, m in DEFAULTS["models"].items():
|
|
240
|
+
if alias not in models and m["provider"] in providers:
|
|
241
|
+
models[alias] = Model(alias, m["model"], m["provider"], m.get("context", 128000),
|
|
242
|
+
m.get("effort"), m.get("fallback"))
|
|
243
|
+
|
|
244
|
+
verify_raw = raw.get("verify") or {}
|
|
245
|
+
verify_checks = verify_raw.get("checks")
|
|
246
|
+
if verify_checks is not None:
|
|
247
|
+
verify_checks = [str(c) for c in verify_checks]
|
|
248
|
+
|
|
249
|
+
hooks_raw = raw.get("hooks") or {}
|
|
250
|
+
hooks_cfg: dict[str, list[HookRule]] = {}
|
|
251
|
+
for stage in ("pre_tool", "post_tool"):
|
|
252
|
+
hooks_cfg[stage] = [HookRule(tools=list(entry.get("tools", [])), command=entry["command"])
|
|
253
|
+
for entry in (hooks_raw.get(stage) or [])]
|
|
254
|
+
|
|
255
|
+
cfg = Config(models=models, providers=providers,
|
|
256
|
+
verify_auto=bool(verify_raw.get("auto", True)),
|
|
257
|
+
verify_checks=verify_checks,
|
|
258
|
+
review_auto=bool(raw.get("review_auto", True)),
|
|
259
|
+
hooks=hooks_cfg)
|
|
260
|
+
|
|
261
|
+
roles: dict[str, Role] = {}
|
|
262
|
+
for name, r in raw["roles"].items():
|
|
263
|
+
if "alias" in r:
|
|
264
|
+
roles[name] = cfg.model(r["alias"])
|
|
265
|
+
else:
|
|
266
|
+
roles[name] = Role(r["model"], providers[r["provider"]], r.get("context", 128000), r.get("effort"))
|
|
267
|
+
cfg.roles = roles
|
|
268
|
+
return cfg
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
def mcp_names() -> list[str]:
|
|
272
|
+
if not CONFIG_PATH.exists():
|
|
273
|
+
return []
|
|
274
|
+
import json as _json
|
|
275
|
+
return list(_json.loads(_strip_jsonc(CONFIG_PATH.read_text())).get("mcp", {}))
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
def mcp_config() -> dict[str, dict[str, Any]]:
|
|
279
|
+
"""The omega-owned "mcp" block as written on disk -- unlike mcp.discover(),
|
|
280
|
+
this never mixes in Claude Code's servers, so the connections manager can
|
|
281
|
+
tell "configured in omega" apart from "merely importable"."""
|
|
282
|
+
if not CONFIG_PATH.exists():
|
|
283
|
+
return {}
|
|
284
|
+
raw: dict[str, Any] = json.loads(_strip_jsonc(CONFIG_PATH.read_text())).get("mcp", {})
|
|
285
|
+
return raw
|
omega/eval/__init__.py
ADDED
omega/eval/cli.py
ADDED
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import time
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
from .. import config
|
|
6
|
+
from ..ui import plain
|
|
7
|
+
from . import runner
|
|
8
|
+
from .report import Report, load_report, new_run_dir, render_table, write_report
|
|
9
|
+
from .report import compare as compare_reports
|
|
10
|
+
from .tasks import TaskError, init_examples, load_tasks
|
|
11
|
+
|
|
12
|
+
console = plain.console
|
|
13
|
+
|
|
14
|
+
_USAGE = """omega eval -- headless task-suite runner
|
|
15
|
+
|
|
16
|
+
usage:
|
|
17
|
+
omega eval init copy example tasks into .omega/evals/
|
|
18
|
+
omega eval run [path] [flags] run tasks (default: .omega/evals/*.yaml)
|
|
19
|
+
omega eval compare <runA> <runB> diff two reports (run id, dir, or report.json path)
|
|
20
|
+
|
|
21
|
+
flags for `run`:
|
|
22
|
+
--models a,b,c model aliases to run against (default: the `main` role)
|
|
23
|
+
--repeat N repeat each task N times (default 1)
|
|
24
|
+
--jobs N max concurrent runs (default 1)
|
|
25
|
+
--json print the report as JSON instead of a table"""
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
async def main(argv: list[str]) -> None:
|
|
29
|
+
if not argv or argv[0] in ("-h", "--help"):
|
|
30
|
+
return console.print(_USAGE, markup=False, highlight=False)
|
|
31
|
+
sub, rest = argv[0], argv[1:]
|
|
32
|
+
|
|
33
|
+
if sub == "init":
|
|
34
|
+
dest = Path.cwd() / ".omega" / "evals"
|
|
35
|
+
written = init_examples(dest)
|
|
36
|
+
console.print(f"[green]wrote {len(written)} example task(s) to {dest}[/green]")
|
|
37
|
+
for p in written:
|
|
38
|
+
console.print(f" {p.name}")
|
|
39
|
+
return
|
|
40
|
+
|
|
41
|
+
if sub == "run":
|
|
42
|
+
return await _run(rest)
|
|
43
|
+
|
|
44
|
+
if sub == "compare":
|
|
45
|
+
return _compare(rest)
|
|
46
|
+
|
|
47
|
+
console.print(f"[red]unknown `omega eval {sub}`[/red] -- init, run, compare")
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _parse_run_args(rest: list[str]) -> tuple[str | None, str | None, int, int, bool]:
|
|
51
|
+
models_arg: str | None = None
|
|
52
|
+
repeat, jobs, as_json = 1, 1, False
|
|
53
|
+
positional: list[str] = []
|
|
54
|
+
i = 0
|
|
55
|
+
while i < len(rest):
|
|
56
|
+
a = rest[i]
|
|
57
|
+
if a == "--models" and i + 1 < len(rest):
|
|
58
|
+
models_arg, i = rest[i + 1], i + 2
|
|
59
|
+
elif a == "--repeat" and i + 1 < len(rest):
|
|
60
|
+
repeat, i = int(rest[i + 1]), i + 2
|
|
61
|
+
elif a == "--jobs" and i + 1 < len(rest):
|
|
62
|
+
jobs, i = int(rest[i + 1]), i + 2
|
|
63
|
+
elif a == "--json":
|
|
64
|
+
as_json, i = True, i + 1
|
|
65
|
+
else:
|
|
66
|
+
positional.append(a)
|
|
67
|
+
i += 1
|
|
68
|
+
path = positional[0] if positional else None
|
|
69
|
+
return path, models_arg, repeat, jobs, as_json
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
async def _run(rest: list[str]) -> None:
|
|
73
|
+
path, models_arg, repeat, jobs, as_json = _parse_run_args(rest)
|
|
74
|
+
try:
|
|
75
|
+
tasks = load_tasks(path)
|
|
76
|
+
except TaskError as e:
|
|
77
|
+
return console.print(f"[red]{e}[/red]")
|
|
78
|
+
if not tasks:
|
|
79
|
+
return console.print("[dim]no eval tasks found -- run `omega eval init` first[/dim]")
|
|
80
|
+
|
|
81
|
+
cfg = config.load()
|
|
82
|
+
try:
|
|
83
|
+
roles = runner.resolve_models(cfg, models_arg)
|
|
84
|
+
except KeyError as e:
|
|
85
|
+
return console.print(f"[red]{e}[/red]")
|
|
86
|
+
|
|
87
|
+
if not as_json:
|
|
88
|
+
console.print(f"[dim]running {len(tasks)} task(s) x {len(roles)} model(s) "
|
|
89
|
+
f"x {repeat} repeat(s)…[/dim]")
|
|
90
|
+
results = await runner.run_suite(cfg, tasks, roles, repeat=repeat, jobs=jobs)
|
|
91
|
+
report = Report(created=time.time(), results=tuple(results))
|
|
92
|
+
|
|
93
|
+
run_dir = new_run_dir(Path.cwd() / ".omega" / "evals" / "runs")
|
|
94
|
+
out_path = write_report(report, run_dir)
|
|
95
|
+
|
|
96
|
+
if as_json:
|
|
97
|
+
console.print(json.dumps(report.to_dict(), indent=1))
|
|
98
|
+
else:
|
|
99
|
+
console.print(render_table(report))
|
|
100
|
+
console.print(f"\n[dim]report saved to {out_path}[/dim]")
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def _resolve_run_path(ref: str) -> Path:
|
|
104
|
+
p = Path(ref)
|
|
105
|
+
if p.is_file():
|
|
106
|
+
return p
|
|
107
|
+
if p.is_dir():
|
|
108
|
+
return p / "report.json"
|
|
109
|
+
runs_root = Path.cwd() / ".omega" / "evals" / "runs"
|
|
110
|
+
candidate = runs_root / ref / "report.json"
|
|
111
|
+
if candidate.exists():
|
|
112
|
+
return candidate
|
|
113
|
+
matches = sorted(runs_root.glob(f"{ref}*/report.json")) if runs_root.exists() else []
|
|
114
|
+
if matches:
|
|
115
|
+
return matches[-1]
|
|
116
|
+
raise TaskError(f"no report found for {ref!r}")
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def _compare(rest: list[str]) -> None:
|
|
120
|
+
if len(rest) < 2:
|
|
121
|
+
return console.print("[red]usage: omega eval compare <runA> <runB>[/red]")
|
|
122
|
+
try:
|
|
123
|
+
a = load_report(_resolve_run_path(rest[0]))
|
|
124
|
+
b = load_report(_resolve_run_path(rest[1]))
|
|
125
|
+
except (TaskError, FileNotFoundError) as e:
|
|
126
|
+
return console.print(f"[red]{e}[/red]")
|
|
127
|
+
console.print(compare_reports(a, b))
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
name: plan-version-flag
|
|
2
|
+
prompt: >
|
|
3
|
+
Investigate omega's CLI entry point (omega/__main__.py) and produce a plan
|
|
4
|
+
for adding a `--version` flag that prints the installed package version.
|
|
5
|
+
Name every real file you would touch and the exact function or block
|
|
6
|
+
you would change in each. Do not write any code -- you are read-only.
|
|
7
|
+
repo: .
|
|
8
|
+
check: "grep -q 'omega/__main__.py' TRANSCRIPT.md"
|
|
9
|
+
timeout_s: 300
|
|
10
|
+
mode: plan
|
|
11
|
+
tags: [planning]
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
name: relative-age-negative-delta
|
|
2
|
+
prompt: >
|
|
3
|
+
In omega/gitlog.py, `_relative_age(seconds)` clamps a negative delta to
|
|
4
|
+
zero with `max(0.0, seconds)`, which silently reports a commit timestamp
|
|
5
|
+
from the future as "0s ago" instead of surfacing the clock skew. Change it
|
|
6
|
+
to handle negative deltas explicitly (your choice of representation -- e.g.
|
|
7
|
+
a leading sign or "in the future"), and add a test in tests/test_gitlog.py
|
|
8
|
+
that pins the behavior you chose. Keep the rest of the function and the
|
|
9
|
+
existing tests in tests/test_gitlog.py passing.
|
|
10
|
+
repo: .
|
|
11
|
+
check: "uv run pytest tests/test_gitlog.py -q"
|
|
12
|
+
timeout_s: 600
|
|
13
|
+
mode: build
|
|
14
|
+
tags: [regression, unit-test]
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
name: version-flag
|
|
2
|
+
prompt: >
|
|
3
|
+
Add a `--version` flag to omega's CLI: `omega --version` should print the
|
|
4
|
+
installed package version and exit without starting an agent turn. Wire it
|
|
5
|
+
into omega/__main__.py next to the other top-level flags.
|
|
6
|
+
repo: .
|
|
7
|
+
check: "uv run omega --version | grep -Eq '[0-9]+\\.[0-9]+\\.[0-9]+'"
|
|
8
|
+
timeout_s: 600
|
|
9
|
+
mode: build
|
|
10
|
+
tags: [cli, smoke]
|
omega/eval/manifest.py
ADDED
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
from dataclasses import dataclass
|
|
2
|
+
from typing import Any
|
|
3
|
+
|
|
4
|
+
from .. import compact, events
|
|
5
|
+
from ..session import Message
|
|
6
|
+
|
|
7
|
+
VOLATILE_MARKER = "<!-- volatile -->"
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
@dataclass(frozen=True)
|
|
11
|
+
class ToolCallRecord:
|
|
12
|
+
name: str
|
|
13
|
+
duration_s: float
|
|
14
|
+
result_chars: int
|
|
15
|
+
|
|
16
|
+
def to_dict(self) -> dict[str, Any]:
|
|
17
|
+
return {"name": self.name, "duration_s": round(self.duration_s, 3),
|
|
18
|
+
"result_chars": self.result_chars}
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@dataclass(frozen=True)
|
|
22
|
+
class RoundRecord:
|
|
23
|
+
index: int
|
|
24
|
+
prompt_tokens: int | None
|
|
25
|
+
completion_tokens: int | None
|
|
26
|
+
cache_tokens: int
|
|
27
|
+
estimated_tokens: int
|
|
28
|
+
tool_calls: tuple[ToolCallRecord, ...]
|
|
29
|
+
|
|
30
|
+
def to_dict(self) -> dict[str, Any]:
|
|
31
|
+
return {
|
|
32
|
+
"index": self.index, "prompt_tokens": self.prompt_tokens,
|
|
33
|
+
"completion_tokens": self.completion_tokens, "cache_tokens": self.cache_tokens,
|
|
34
|
+
"estimated_tokens": self.estimated_tokens,
|
|
35
|
+
"tool_calls": [c.to_dict() for c in self.tool_calls],
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
@dataclass(frozen=True)
|
|
40
|
+
class DriftPoint:
|
|
41
|
+
round: int
|
|
42
|
+
estimated: int
|
|
43
|
+
actual: int
|
|
44
|
+
|
|
45
|
+
@property
|
|
46
|
+
def drift(self) -> int:
|
|
47
|
+
return self.actual - self.estimated
|
|
48
|
+
|
|
49
|
+
@property
|
|
50
|
+
def drift_pct(self) -> float | None:
|
|
51
|
+
return (self.drift / self.actual) if self.actual else None
|
|
52
|
+
|
|
53
|
+
def to_dict(self) -> dict[str, Any]:
|
|
54
|
+
return {"round": self.round, "estimated": self.estimated, "actual": self.actual,
|
|
55
|
+
"drift": self.drift, "drift_pct": self.drift_pct}
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
@dataclass(frozen=True)
|
|
59
|
+
class SystemZones:
|
|
60
|
+
fixed_chars: int
|
|
61
|
+
volatile_chars: int
|
|
62
|
+
|
|
63
|
+
def to_dict(self) -> dict[str, Any]:
|
|
64
|
+
return {"fixed_chars": self.fixed_chars, "volatile_chars": self.volatile_chars}
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
@dataclass(frozen=True)
|
|
68
|
+
class ContextManifest:
|
|
69
|
+
rounds: tuple[RoundRecord, ...]
|
|
70
|
+
system_zones: SystemZones
|
|
71
|
+
drift: tuple[DriftPoint, ...]
|
|
72
|
+
|
|
73
|
+
def to_dict(self) -> dict[str, Any]:
|
|
74
|
+
return {
|
|
75
|
+
"rounds": [r.to_dict() for r in self.rounds],
|
|
76
|
+
"system_zones": self.system_zones.to_dict(),
|
|
77
|
+
"drift": [d.to_dict() for d in self.drift],
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def split_system_zones(system: str, marker: str = VOLATILE_MARKER) -> SystemZones:
|
|
82
|
+
if marker not in system:
|
|
83
|
+
return SystemZones(fixed_chars=len(system), volatile_chars=0)
|
|
84
|
+
fixed, _, volatile = system.partition(marker)
|
|
85
|
+
return SystemZones(fixed_chars=len(fixed), volatile_chars=len(volatile))
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _cache_tokens(usage: events.Usage) -> int:
|
|
89
|
+
# `cached_tokens`/`cache_read` may not exist on every Usage build -- A2 is
|
|
90
|
+
# adding cache fields to this event independently of this module.
|
|
91
|
+
return int(getattr(usage, "cached_tokens", None) or getattr(usage, "cache_read", None) or 0)
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def build_manifest(evs: list[events.Event], system: str, schemas: list[dict[str, Any]],
|
|
95
|
+
initial_history: list[Message], final_history: list[Message]) -> ContextManifest:
|
|
96
|
+
"""Reconstructs per-round token/tool telemetry from the events emitted by
|
|
97
|
+
`loop.run_agent` plus the `history` list it mutates in place. Each round
|
|
98
|
+
appends exactly 1 (assistant) + len(tool_calls) messages to `history`
|
|
99
|
+
before the next request goes out -- except the final round (no tool
|
|
100
|
+
calls), which ends in `Done` instead of `Usage` and so has no actual
|
|
101
|
+
`prompt_tokens` in the current event schema; its drift point is skipped,
|
|
102
|
+
though the round itself is still recorded with `prompt_tokens=None`."""
|
|
103
|
+
overhead = compact.estimate_tokens([{"role": "system", "content": system}]) \
|
|
104
|
+
+ compact.estimate_tokens(schemas)
|
|
105
|
+
|
|
106
|
+
rounds: list[RoundRecord] = []
|
|
107
|
+
drift: list[DriftPoint] = []
|
|
108
|
+
msg_idx = len(initial_history)
|
|
109
|
+
round_calls: list[ToolCallRecord] = []
|
|
110
|
+
round_index = -1
|
|
111
|
+
|
|
112
|
+
for ev in evs:
|
|
113
|
+
if isinstance(ev, events.Phase) and ev.state == "waiting":
|
|
114
|
+
round_index += 1
|
|
115
|
+
round_calls = []
|
|
116
|
+
elif isinstance(ev, events.ToolEnd):
|
|
117
|
+
round_calls.append(ToolCallRecord(ev.name, ev.duration_s, ev.result_chars))
|
|
118
|
+
elif isinstance(ev, events.Usage):
|
|
119
|
+
estimated = overhead + compact.estimate_tokens(final_history[:msg_idx])
|
|
120
|
+
rounds.append(RoundRecord(round_index, ev.prompt_tokens, ev.completion_tokens,
|
|
121
|
+
_cache_tokens(ev), estimated, tuple(round_calls)))
|
|
122
|
+
if ev.prompt_tokens:
|
|
123
|
+
drift.append(DriftPoint(round_index, estimated, ev.prompt_tokens))
|
|
124
|
+
msg_idx += 1 + len(round_calls)
|
|
125
|
+
elif isinstance(ev, events.Done):
|
|
126
|
+
estimated = overhead + compact.estimate_tokens(final_history[:msg_idx])
|
|
127
|
+
rounds.append(RoundRecord(round_index, None, None, 0, estimated, tuple(round_calls)))
|
|
128
|
+
|
|
129
|
+
return ContextManifest(tuple(rounds), split_system_zones(system), tuple(drift))
|
omega/eval/prices.py
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# Per-million-token USD prices (input, output), keyed by the model catalog
|
|
2
|
+
# alias (config.DEFAULTS["models"]) -- not the provider's raw model id, so a
|
|
3
|
+
# `--models opus,sonnet` run can price itself without a network round trip.
|
|
4
|
+
PRICES: dict[str, tuple[float, float]] = {
|
|
5
|
+
"fable": (10.0, 50.0),
|
|
6
|
+
"opus": (5.0, 25.0),
|
|
7
|
+
"sonnet": (2.0, 10.0),
|
|
8
|
+
"haiku": (1.0, 5.0),
|
|
9
|
+
"spark": (1.25, 4.25),
|
|
10
|
+
"kimi": (3.0, 15.0),
|
|
11
|
+
"glm": (0.6, 2.2),
|
|
12
|
+
"astra": (10.0, 50.0),
|
|
13
|
+
"sol": (2.0, 10.0),
|
|
14
|
+
"terra": (2.0, 12.0),
|
|
15
|
+
"luna": (0.2, 1.2),
|
|
16
|
+
"codex": (1.75, 14.0),
|
|
17
|
+
"grok": (2.0, 6.0),
|
|
18
|
+
"grok-build": (1.0, 2.0),
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def estimate_cost(alias: str, tokens_in: int, tokens_out: int) -> float | None:
|
|
23
|
+
"""None for an alias absent from PRICES -- an unpriced model must show up
|
|
24
|
+
as "unknown" in a report, never as a silent $0."""
|
|
25
|
+
prices = PRICES.get(alias)
|
|
26
|
+
if prices is None:
|
|
27
|
+
return None
|
|
28
|
+
price_in, price_out = prices
|
|
29
|
+
return tokens_in / 1_000_000 * price_in + tokens_out / 1_000_000 * price_out
|