trap-cli 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- trap/__init__.py +6 -0
- trap/_version.py +24 -0
- trap/auth/__init__.py +15 -0
- trap/auth/client.py +56 -0
- trap/auth/login.py +39 -0
- trap/auth/oauth.py +82 -0
- trap/auth/store.py +39 -0
- trap/cli/__init__.py +274 -0
- trap/cli/_auth.py +103 -0
- trap/cost/__init__.py +3 -0
- trap/cost/calculator.py +41 -0
- trap/cost/providers.py +122 -0
- trap/cost/proxy.py +186 -0
- trap/display/__init__.py +4 -0
- trap/display/progress.py +61 -0
- trap/display/submit.py +20 -0
- trap/environment/__init__.py +3 -0
- trap/environment/detector.py +81 -0
- trap/git_ops/__init__.py +13 -0
- trap/git_ops/base.py +10 -0
- trap/git_ops/local.py +60 -0
- trap/git_ops/remote.py +63 -0
- trap/git_ops/rev.py +119 -0
- trap/git_ops/url.py +98 -0
- trap/loader/__init__.py +5 -0
- trap/loader/errors.py +7 -0
- trap/loader/trap_yaml.py +99 -0
- trap/loader/traptask_yaml.py +85 -0
- trap/models/__init__.py +36 -0
- trap/models/cost.py +36 -0
- trap/models/environment.py +24 -0
- trap/models/provenance.py +20 -0
- trap/models/report.py +61 -0
- trap/models/results.py +17 -0
- trap/models/trap_yaml.py +64 -0
- trap/models/traptask_yaml.py +58 -0
- trap/report/__init__.py +19 -0
- trap/report/base.py +8 -0
- trap/report/handle.py +54 -0
- trap/report/json.py +11 -0
- trap/report/rich.py +116 -0
- trap/runner/__init__.py +5 -0
- trap/runner/capture.py +34 -0
- trap/runner/grader.py +36 -0
- trap/runner/judge.py +50 -0
- trap/runner/layout.py +36 -0
- trap/runner/proc.py +122 -0
- trap/runner/solution.py +80 -0
- trap/runner/task.py +102 -0
- trap_cli-0.0.1.dist-info/METADATA +88 -0
- trap_cli-0.0.1.dist-info/RECORD +54 -0
- trap_cli-0.0.1.dist-info/WHEEL +4 -0
- trap_cli-0.0.1.dist-info/entry_points.txt +2 -0
- trap_cli-0.0.1.dist-info/licenses/LICENSE +21 -0
trap/cost/calculator.py
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
"""Cost calculation for LLM token usage.
|
|
2
|
+
|
|
3
|
+
Isolated so this logic can be moved server-side without touching the proxy infrastructure.
|
|
4
|
+
The only public symbol is :func:`calculate_call_cost`.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from tokencost import TOKEN_COSTS as _TOKEN_COSTS # type: ignore[import-untyped]
|
|
10
|
+
from tokencost import calculate_cost_by_tokens as _calc_cost # type: ignore[import-untyped]
|
|
11
|
+
from tokencost import register_model_pattern as _register_pattern # type: ignore[import-untyped]
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _register_anthropic_version_patterns() -> None:
|
|
15
|
+
# Anthropic now returns version-based names (e.g. "claude-sonnet-4-6") but tokencost only
|
|
16
|
+
# has date-based entries (e.g. "claude-sonnet-4-20250514"). Register wildcard patterns so
|
|
17
|
+
# calculate_cost_by_tokens can resolve them via _normalize_model_for_pricing.
|
|
18
|
+
for pattern, canonical in [
|
|
19
|
+
("claude-sonnet-4-*", "claude-sonnet-4-20250514"),
|
|
20
|
+
("claude-opus-4-*", "claude-opus-4-20250514"),
|
|
21
|
+
]:
|
|
22
|
+
entry = _TOKEN_COSTS.get(canonical)
|
|
23
|
+
if entry: # pragma: no branch - both canonicals ship in tokencost
|
|
24
|
+
_register_pattern(
|
|
25
|
+
pattern,
|
|
26
|
+
float(entry["input_cost_per_token"]) * 1000,
|
|
27
|
+
float(entry["output_cost_per_token"]) * 1000,
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
_register_anthropic_version_patterns()
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def calculate_call_cost(prompt_tokens: int, completion_tokens: int, model: str) -> float:
|
|
35
|
+
"""Return the USD cost for one API call. Returns 0.0 if the model is not in the pricing table."""
|
|
36
|
+
try:
|
|
37
|
+
return float(
|
|
38
|
+
_calc_cost(prompt_tokens, model, "input") + _calc_cost(completion_tokens, model, "output")
|
|
39
|
+
)
|
|
40
|
+
except KeyError:
|
|
41
|
+
return 0.0
|
trap/cost/providers.py
ADDED
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import enum
|
|
4
|
+
import json
|
|
5
|
+
import os
|
|
6
|
+
from dataclasses import dataclass
|
|
7
|
+
|
|
8
|
+
# -- Provider registry --------------------------------------------------------
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class _ProtocolStyle(enum.StrEnum):
|
|
12
|
+
ANTHROPIC_COMPATIBLE = "anthropic-compatible"
|
|
13
|
+
OPENAI_COMPATIBLE = "openai-compatible"
|
|
14
|
+
|
|
15
|
+
def parse(self, content_type: str, body: bytes) -> tuple[int, int, str | None]:
|
|
16
|
+
"""Extract (prompt_tokens, completion_tokens, model) from an API response."""
|
|
17
|
+
is_streaming = "text/event-stream" in content_type
|
|
18
|
+
match (self, is_streaming):
|
|
19
|
+
case (_ProtocolStyle.ANTHROPIC_COMPATIBLE, True):
|
|
20
|
+
return self._parse_anthropic_style_sse(body.decode("utf-8", errors="replace"))
|
|
21
|
+
case (_ProtocolStyle.ANTHROPIC_COMPATIBLE, False):
|
|
22
|
+
return self._parse_json(body, "input_tokens", "output_tokens")
|
|
23
|
+
case (_ProtocolStyle.OPENAI_COMPATIBLE, True):
|
|
24
|
+
return self._parse_openai_style_sse(body.decode("utf-8", errors="replace"))
|
|
25
|
+
case (_ProtocolStyle.OPENAI_COMPATIBLE, False):
|
|
26
|
+
return self._parse_json(body, "prompt_tokens", "completion_tokens")
|
|
27
|
+
case _: # pragma: no cover - exhaustive over the two styles above
|
|
28
|
+
raise ValueError(f"Unsupported protocol style: {self!r}")
|
|
29
|
+
|
|
30
|
+
@staticmethod
|
|
31
|
+
def _parse_json(body: bytes, prompt_key: str, completion_key: str) -> tuple[int, int, str | None]:
|
|
32
|
+
try:
|
|
33
|
+
data = json.loads(body)
|
|
34
|
+
except (json.JSONDecodeError, ValueError):
|
|
35
|
+
return 0, 0, None
|
|
36
|
+
usage = data.get("usage", {})
|
|
37
|
+
return usage.get(prompt_key, 0), usage.get(completion_key, 0), data.get("model")
|
|
38
|
+
|
|
39
|
+
@staticmethod
|
|
40
|
+
def _parse_anthropic_style_sse(text: str) -> tuple[int, int, str | None]:
|
|
41
|
+
prompt_tokens = 0
|
|
42
|
+
completion_tokens = 0
|
|
43
|
+
model = None
|
|
44
|
+
for line in text.splitlines():
|
|
45
|
+
if not line.startswith("data: "):
|
|
46
|
+
continue
|
|
47
|
+
try:
|
|
48
|
+
event = json.loads(line[6:])
|
|
49
|
+
except (json.JSONDecodeError, ValueError):
|
|
50
|
+
continue
|
|
51
|
+
if event.get("type") == "message_start":
|
|
52
|
+
msg = event.get("message", {})
|
|
53
|
+
usage = msg.get("usage", {})
|
|
54
|
+
prompt_tokens += usage.get("input_tokens", 0)
|
|
55
|
+
model = model or msg.get("model")
|
|
56
|
+
elif event.get("type") == "message_delta":
|
|
57
|
+
completion_tokens += event.get("usage", {}).get("output_tokens", 0)
|
|
58
|
+
return prompt_tokens, completion_tokens, model
|
|
59
|
+
|
|
60
|
+
@staticmethod
|
|
61
|
+
def _parse_openai_style_sse(text: str) -> tuple[int, int, str | None]:
|
|
62
|
+
for line in text.splitlines():
|
|
63
|
+
if not line.startswith("data: ") or "[DONE]" in line:
|
|
64
|
+
continue
|
|
65
|
+
try:
|
|
66
|
+
chunk = json.loads(line[6:])
|
|
67
|
+
except (json.JSONDecodeError, ValueError):
|
|
68
|
+
continue
|
|
69
|
+
usage = chunk.get("usage")
|
|
70
|
+
if usage:
|
|
71
|
+
return usage.get("prompt_tokens", 0), usage.get("completion_tokens", 0), chunk.get("model")
|
|
72
|
+
return 0, 0, None
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
@dataclass(frozen=True)
|
|
76
|
+
class _ProviderConfig:
|
|
77
|
+
key_env: str # env var name for the API key
|
|
78
|
+
base_env: str # env var name for the base URL override
|
|
79
|
+
upstream: str # default upstream base URL
|
|
80
|
+
style: _ProtocolStyle # request/response format used by this provider
|
|
81
|
+
always_intercept: bool = False # True for OAuth-based tools that set no API key env var
|
|
82
|
+
|
|
83
|
+
def resolve_upstream(self) -> str:
|
|
84
|
+
"""Return the effective upstream URL, honouring any user-set env override."""
|
|
85
|
+
return os.environ.get(self.base_env) or self.upstream
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
# Central registry of supported LLM providers.
|
|
89
|
+
#
|
|
90
|
+
# To add a new provider: add an entry to _CONFIGS. Set always_intercept=True for
|
|
91
|
+
# OAuth-based tools (Claude Code, Codex CLI, …) that set no API key env var but
|
|
92
|
+
# still respect the base URL override.
|
|
93
|
+
_CONFIGS: dict[str, _ProviderConfig] = {
|
|
94
|
+
"anthropic": _ProviderConfig(
|
|
95
|
+
"ANTHROPIC_API_KEY",
|
|
96
|
+
"ANTHROPIC_BASE_URL",
|
|
97
|
+
"https://api.anthropic.com",
|
|
98
|
+
style=_ProtocolStyle.ANTHROPIC_COMPATIBLE,
|
|
99
|
+
always_intercept=True,
|
|
100
|
+
),
|
|
101
|
+
"openai": _ProviderConfig(
|
|
102
|
+
"OPENAI_API_KEY",
|
|
103
|
+
"OPENAI_BASE_URL",
|
|
104
|
+
"https://api.openai.com/v1",
|
|
105
|
+
style=_ProtocolStyle.OPENAI_COMPATIBLE,
|
|
106
|
+
always_intercept=True,
|
|
107
|
+
),
|
|
108
|
+
"mistral": _ProviderConfig(
|
|
109
|
+
"MISTRAL_API_KEY",
|
|
110
|
+
"MISTRAL_BASE_URL",
|
|
111
|
+
"https://api.mistral.ai",
|
|
112
|
+
style=_ProtocolStyle.OPENAI_COMPATIBLE,
|
|
113
|
+
),
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def active_provider_configs() -> dict[str, _ProviderConfig]:
|
|
118
|
+
"""Providers active in this environment: those with their API key env var set,
|
|
119
|
+
plus the always-intercept ones (OAuth tools that honour the base URL override)."""
|
|
120
|
+
return {
|
|
121
|
+
name: cfg for name, cfg in _CONFIGS.items() if os.environ.get(cfg.key_env) or cfg.always_intercept
|
|
122
|
+
}
|
trap/cost/proxy.py
ADDED
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import http.server
|
|
4
|
+
import socketserver
|
|
5
|
+
import threading
|
|
6
|
+
from functools import cached_property
|
|
7
|
+
from typing import TYPE_CHECKING
|
|
8
|
+
|
|
9
|
+
import httpx
|
|
10
|
+
|
|
11
|
+
from trap.cost.calculator import calculate_call_cost as _calc_cost
|
|
12
|
+
from trap.cost.providers import active_provider_configs
|
|
13
|
+
from trap.models.cost import CaseCost, ModelCost
|
|
14
|
+
|
|
15
|
+
if TYPE_CHECKING:
|
|
16
|
+
from trap.cost.providers import _ProviderConfig
|
|
17
|
+
|
|
18
|
+
# Three-class structure imposed by the socketserver framework:
|
|
19
|
+
#
|
|
20
|
+
# CostProxy →(creates one per active provider)→ _ProxyServer
|
|
21
|
+
# │ .proxy = cost_proxy (back-ref for bookkeeping)
|
|
22
|
+
# └─→(framework)→ _ProxyHandler
|
|
23
|
+
# │ .server = proxy_server
|
|
24
|
+
#
|
|
25
|
+
# Each _ProxyServer is bound to its own port and knows its provider, protocol
|
|
26
|
+
# style, and upstream URL — no per-request provider detection needed.
|
|
27
|
+
# _ProxyHandler is ephemeral (one per request); _ProxyServer and CostProxy
|
|
28
|
+
# live for the duration of a case run.
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class CostProxy:
|
|
32
|
+
"""HTTP reverse proxy that intercepts LLM API calls to track token usage and cost per case."""
|
|
33
|
+
|
|
34
|
+
def __init__(self) -> None:
|
|
35
|
+
self._cost_buckets: dict[tuple[str, str | None], ModelCost] = {}
|
|
36
|
+
self._lock = threading.Lock()
|
|
37
|
+
# Servers are created here to bind ports immediately; threads start in start().
|
|
38
|
+
self._servers: dict[str, _ProxyServer] = {
|
|
39
|
+
name: _ProxyServer(self, name, cfg) for name, cfg in active_provider_configs().items()
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
@cached_property
|
|
43
|
+
def env_overrides(self) -> dict[str, str]:
|
|
44
|
+
"""Env vars pointing each provider SDK at its proxy port."""
|
|
45
|
+
return {
|
|
46
|
+
server.base_env: f"http://127.0.0.1:{server.server_address[1]}"
|
|
47
|
+
for server in self._servers.values()
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
def start(self) -> None:
|
|
51
|
+
"""Start serving threads for all provider proxy servers."""
|
|
52
|
+
for server in self._servers.values():
|
|
53
|
+
threading.Thread(target=server.serve_forever, daemon=True).start()
|
|
54
|
+
|
|
55
|
+
def stop(self) -> CaseCost:
|
|
56
|
+
"""Shut down all proxy servers and return accumulated cost data."""
|
|
57
|
+
for server in self._servers.values():
|
|
58
|
+
server.shutdown()
|
|
59
|
+
with self._lock:
|
|
60
|
+
self._servers.clear()
|
|
61
|
+
return CaseCost(by_model=list(self._cost_buckets.values()))
|
|
62
|
+
|
|
63
|
+
def _accumulate(
|
|
64
|
+
self, provider: str, prompt_tokens: int, completion_tokens: int, model: str | None
|
|
65
|
+
) -> None:
|
|
66
|
+
if not (prompt_tokens or completion_tokens):
|
|
67
|
+
return
|
|
68
|
+
call_cost = _calc_cost(prompt_tokens, completion_tokens, model) if model else 0.0
|
|
69
|
+
with self._lock:
|
|
70
|
+
key = (provider, model)
|
|
71
|
+
entry = self._cost_buckets.get(key)
|
|
72
|
+
if entry is None:
|
|
73
|
+
self._cost_buckets[key] = ModelCost(
|
|
74
|
+
provider=provider,
|
|
75
|
+
model=model,
|
|
76
|
+
prompt_tokens=prompt_tokens,
|
|
77
|
+
completion_tokens=completion_tokens,
|
|
78
|
+
cost_usd=call_cost,
|
|
79
|
+
calls=1,
|
|
80
|
+
)
|
|
81
|
+
else:
|
|
82
|
+
entry.prompt_tokens += prompt_tokens
|
|
83
|
+
entry.completion_tokens += completion_tokens
|
|
84
|
+
entry.cost_usd += call_cost
|
|
85
|
+
entry.calls += 1
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
class _ProxyServer(socketserver.ThreadingMixIn, socketserver.TCPServer):
|
|
89
|
+
allow_reuse_address = True
|
|
90
|
+
daemon_threads = True
|
|
91
|
+
block_on_close = False
|
|
92
|
+
|
|
93
|
+
def __init__(self, proxy: CostProxy, provider: str, cfg: _ProviderConfig) -> None:
|
|
94
|
+
self.proxy = proxy
|
|
95
|
+
self.provider = provider
|
|
96
|
+
self.base_env = cfg.base_env
|
|
97
|
+
self.style = cfg.style
|
|
98
|
+
# Capture upstream before env_overrides() redirects base_env to the proxy itself.
|
|
99
|
+
self.upstream = cfg.resolve_upstream()
|
|
100
|
+
super().__init__(("127.0.0.1", 0), _ProxyHandler)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
class _ProxyHandler(http.server.BaseHTTPRequestHandler):
|
|
104
|
+
server: _ProxyServer # type: ignore[assignment]
|
|
105
|
+
|
|
106
|
+
_STRIP_REQUEST_HEADERS = frozenset(
|
|
107
|
+
{
|
|
108
|
+
"host",
|
|
109
|
+
"content-length",
|
|
110
|
+
"accept-encoding",
|
|
111
|
+
"connection",
|
|
112
|
+
"keep-alive",
|
|
113
|
+
"proxy-authenticate",
|
|
114
|
+
"proxy-authorization",
|
|
115
|
+
"te",
|
|
116
|
+
"trailers",
|
|
117
|
+
"transfer-encoding",
|
|
118
|
+
"upgrade",
|
|
119
|
+
}
|
|
120
|
+
)
|
|
121
|
+
_STRIP_RESPONSE_HEADERS = frozenset(
|
|
122
|
+
{
|
|
123
|
+
"content-length",
|
|
124
|
+
"content-encoding",
|
|
125
|
+
"transfer-encoding",
|
|
126
|
+
"connection",
|
|
127
|
+
"keep-alive",
|
|
128
|
+
"proxy-authenticate",
|
|
129
|
+
"proxy-authorization",
|
|
130
|
+
"te",
|
|
131
|
+
"trailers",
|
|
132
|
+
"upgrade",
|
|
133
|
+
}
|
|
134
|
+
)
|
|
135
|
+
|
|
136
|
+
def do_POST(self) -> None:
|
|
137
|
+
self._forward()
|
|
138
|
+
|
|
139
|
+
def do_GET(self) -> None:
|
|
140
|
+
self._forward()
|
|
141
|
+
|
|
142
|
+
def _forward(self) -> None:
|
|
143
|
+
proxy = self.server.proxy
|
|
144
|
+
body = self._read_body()
|
|
145
|
+
status_code, content_type, chunks = self._relay_to_upstream(self.server.upstream, body)
|
|
146
|
+
if 0 < status_code < 400:
|
|
147
|
+
prompt_tokens, completion_tokens, model = self.server.style.parse(content_type, b"".join(chunks))
|
|
148
|
+
proxy._accumulate(self.server.provider, prompt_tokens, completion_tokens, model)
|
|
149
|
+
|
|
150
|
+
def _read_body(self) -> bytes:
|
|
151
|
+
n = int(self.headers.get("Content-Length", 0))
|
|
152
|
+
return self.rfile.read(n) if n else b""
|
|
153
|
+
|
|
154
|
+
def _relay_to_upstream(self, upstream: str, body: bytes) -> tuple[int, str, list[bytes]]:
|
|
155
|
+
target_url = upstream.rstrip("/") + self.path
|
|
156
|
+
forward_headers = {
|
|
157
|
+
k: v for k, v in self.headers.items() if k.lower() not in self._STRIP_REQUEST_HEADERS
|
|
158
|
+
}
|
|
159
|
+
status_code = 0
|
|
160
|
+
content_type = ""
|
|
161
|
+
chunks: list[bytes] = []
|
|
162
|
+
try:
|
|
163
|
+
with httpx.Client(timeout=300.0) as client:
|
|
164
|
+
with client.stream(self.command, target_url, content=body, headers=forward_headers) as resp:
|
|
165
|
+
status_code = resp.status_code
|
|
166
|
+
content_type = resp.headers.get("content-type", "")
|
|
167
|
+
self.send_response(status_code)
|
|
168
|
+
for h_name, h_val in resp.headers.multi_items():
|
|
169
|
+
if h_name.lower() not in self._STRIP_RESPONSE_HEADERS:
|
|
170
|
+
self.send_header(h_name, h_val)
|
|
171
|
+
self.end_headers()
|
|
172
|
+
for chunk in resp.iter_bytes():
|
|
173
|
+
try:
|
|
174
|
+
self.wfile.write(chunk)
|
|
175
|
+
self.wfile.flush()
|
|
176
|
+
except (BrokenPipeError, ConnectionResetError): # pragma: no cover - client hangup
|
|
177
|
+
break
|
|
178
|
+
chunks.append(chunk)
|
|
179
|
+
except httpx.HTTPError as exc:
|
|
180
|
+
if not status_code: # pragma: no branch - the set-status case needs a mid-stream failure
|
|
181
|
+
self.send_error(502, str(exc))
|
|
182
|
+
return status_code, content_type, chunks
|
|
183
|
+
|
|
184
|
+
def log_message(self, format: str, *args: object) -> None:
|
|
185
|
+
# Suppress the default per-request stderr logging from BaseHTTPRequestHandler.
|
|
186
|
+
pass
|
trap/display/__init__.py
ADDED
trap/display/progress.py
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from types import TracebackType
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
from rich.console import Console
|
|
7
|
+
from rich.progress import (
|
|
8
|
+
BarColumn,
|
|
9
|
+
MofNCompleteColumn,
|
|
10
|
+
Progress,
|
|
11
|
+
SpinnerColumn,
|
|
12
|
+
TaskProgressColumn,
|
|
13
|
+
TextColumn,
|
|
14
|
+
TimeElapsedColumn,
|
|
15
|
+
)
|
|
16
|
+
from rich.table import Column
|
|
17
|
+
|
|
18
|
+
from trap.models import CaseResult, TraptaskCase
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class CaseProgress:
|
|
22
|
+
"""Context manager that shows a Rich progress bar while cases run.
|
|
23
|
+
|
|
24
|
+
Pass ``console=None`` (the default) for a silent no-op.
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
def __init__(self, cases: tuple[TraptaskCase, ...], *, console: Console | None = None) -> None:
|
|
28
|
+
self._n = len(cases)
|
|
29
|
+
# Always build the Progress; disable=True (console=None) makes every call a
|
|
30
|
+
# no-op — Rich renders nothing rather than falling back to its stdout console.
|
|
31
|
+
self._progress = Progress(
|
|
32
|
+
SpinnerColumn(style="dark_orange"),
|
|
33
|
+
TextColumn("[bold]{task.description}", table_column=Column(width=30, no_wrap=True)),
|
|
34
|
+
BarColumn(complete_style="dark_orange", finished_style="bright_yellow"),
|
|
35
|
+
TaskProgressColumn(),
|
|
36
|
+
MofNCompleteColumn(),
|
|
37
|
+
TimeElapsedColumn(),
|
|
38
|
+
console=console,
|
|
39
|
+
transient=True,
|
|
40
|
+
disable=console is None,
|
|
41
|
+
)
|
|
42
|
+
self._task_id: Any = None
|
|
43
|
+
|
|
44
|
+
def __enter__(self) -> CaseProgress:
|
|
45
|
+
self._progress.start()
|
|
46
|
+
self._task_id = self._progress.add_task("starting...", total=self._n)
|
|
47
|
+
return self
|
|
48
|
+
|
|
49
|
+
def __exit__(
|
|
50
|
+
self,
|
|
51
|
+
exc_type: type[BaseException] | None,
|
|
52
|
+
exc_val: BaseException | None,
|
|
53
|
+
exc_tb: TracebackType | None,
|
|
54
|
+
) -> None:
|
|
55
|
+
self._progress.stop()
|
|
56
|
+
|
|
57
|
+
def on_case_start(self, case_id: str) -> None:
|
|
58
|
+
self._progress.update(self._task_id, description=f"running [bold]{case_id}[/bold]")
|
|
59
|
+
|
|
60
|
+
def on_case_done(self, _: CaseResult) -> None:
|
|
61
|
+
self._progress.advance(self._task_id)
|
trap/display/submit.py
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from rich.console import Console
|
|
4
|
+
from rich.table import Table
|
|
5
|
+
|
|
6
|
+
console = Console()
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def render_submit_result(resp_data: dict) -> None:
|
|
10
|
+
run_obj = resp_data.get("run") or {}
|
|
11
|
+
passed = run_obj.get("passed")
|
|
12
|
+
table = Table.grid(padding=(0, 2))
|
|
13
|
+
table.add_column(style="dim")
|
|
14
|
+
table.add_column()
|
|
15
|
+
table.add_row("status", "[green]✓ passed[/green]" if passed else "[red]✗ failed[/red]")
|
|
16
|
+
table.add_row("run", f"[bold]{run_obj.get('id', '?')}[/bold]")
|
|
17
|
+
table.add_row("score", f"[cyan]{run_obj.get('total_score')}[/cyan]")
|
|
18
|
+
if view_url := resp_data.get("view_url"):
|
|
19
|
+
table.add_row("url", f"[link={view_url}]{view_url}[/link]")
|
|
20
|
+
console.print(table)
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
# Best-effort detection of the host's runtime environment, recorded in the report
|
|
2
|
+
# so runs are comparable across machines. Each probe is decorated with `@_safe`, so
|
|
3
|
+
# a failing call degrades that field to None and never aborts the run.
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import functools
|
|
7
|
+
import platform
|
|
8
|
+
from collections.abc import Callable
|
|
9
|
+
|
|
10
|
+
import cpuinfo
|
|
11
|
+
import psutil
|
|
12
|
+
|
|
13
|
+
from trap.models.environment import Cpu, Environment
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class EnvironmentDetector:
|
|
17
|
+
"""Probes the host machine for a fastfetch-like subset of its environment."""
|
|
18
|
+
|
|
19
|
+
@staticmethod
|
|
20
|
+
def _safe[**P, T](probe: Callable[P, T]) -> Callable[P, T | None]:
|
|
21
|
+
"""Decorate a probe so any failure is swallowed to None — one bad field
|
|
22
|
+
can't sink the whole run."""
|
|
23
|
+
|
|
24
|
+
@functools.wraps(probe)
|
|
25
|
+
def wrapper(*args: P.args, **kwargs: P.kwargs) -> T | None:
|
|
26
|
+
try:
|
|
27
|
+
return probe(*args, **kwargs)
|
|
28
|
+
except Exception:
|
|
29
|
+
return None
|
|
30
|
+
|
|
31
|
+
return wrapper
|
|
32
|
+
|
|
33
|
+
def detect(self) -> Environment:
|
|
34
|
+
return Environment(
|
|
35
|
+
os=self._get_os(),
|
|
36
|
+
kernel=self._get_kernel(),
|
|
37
|
+
arch=self._get_arch(),
|
|
38
|
+
cpu=Cpu(
|
|
39
|
+
model=self._get_cpu_model(),
|
|
40
|
+
cores_physical=self._get_cpu_cores_physical(),
|
|
41
|
+
cores_logical=self._get_cpu_cores_logical(),
|
|
42
|
+
),
|
|
43
|
+
memory_total_bytes=self._get_memory_total_bytes(),
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
# -- probes: stdlib `platform` ------------------------------------------
|
|
47
|
+
|
|
48
|
+
@_safe
|
|
49
|
+
def _get_os(self) -> str | None:
|
|
50
|
+
if platform.system() == "Darwin":
|
|
51
|
+
ver = platform.mac_ver()[0]
|
|
52
|
+
return f"macOS {ver}" if ver else "macOS"
|
|
53
|
+
if platform.system() == "Linux":
|
|
54
|
+
return platform.freedesktop_os_release().get("PRETTY_NAME") or platform.system()
|
|
55
|
+
return platform.system() or None
|
|
56
|
+
|
|
57
|
+
@_safe
|
|
58
|
+
def _get_kernel(self) -> str | None:
|
|
59
|
+
return f"{platform.system()} {platform.release()}".strip() or None
|
|
60
|
+
|
|
61
|
+
@_safe
|
|
62
|
+
def _get_arch(self) -> str | None:
|
|
63
|
+
return platform.machine() or None
|
|
64
|
+
|
|
65
|
+
# -- probes: py-cpuinfo / psutil ----------------------------------------
|
|
66
|
+
|
|
67
|
+
@_safe
|
|
68
|
+
def _get_cpu_model(self) -> str | None:
|
|
69
|
+
return cpuinfo.get_cpu_info().get("brand_raw") or None
|
|
70
|
+
|
|
71
|
+
@_safe
|
|
72
|
+
def _get_cpu_cores_physical(self) -> int | None:
|
|
73
|
+
return psutil.cpu_count(logical=False)
|
|
74
|
+
|
|
75
|
+
@_safe
|
|
76
|
+
def _get_cpu_cores_logical(self) -> int | None:
|
|
77
|
+
return psutil.cpu_count(logical=True)
|
|
78
|
+
|
|
79
|
+
@_safe
|
|
80
|
+
def _get_memory_total_bytes(self) -> int | None:
|
|
81
|
+
return psutil.virtual_memory().total
|
trap/git_ops/__init__.py
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from trap.git_ops.base import GitOpsError
|
|
4
|
+
from trap.git_ops.local import LocalRepo
|
|
5
|
+
from trap.git_ops.remote import RemoteRepo
|
|
6
|
+
from trap.git_ops.url import ParsedGitUrl
|
|
7
|
+
|
|
8
|
+
__all__ = [
|
|
9
|
+
"GitOpsError",
|
|
10
|
+
"LocalRepo",
|
|
11
|
+
"ParsedGitUrl",
|
|
12
|
+
"RemoteRepo",
|
|
13
|
+
]
|
trap/git_ops/base.py
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from collections.abc import Callable
|
|
4
|
+
|
|
5
|
+
# Progress callback used across clone/fetch/update steps; None disables progress output.
|
|
6
|
+
ProgressCallback = Callable[[str], None] | None
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class GitOpsError(Exception):
|
|
10
|
+
pass
|
trap/git_ops/local.py
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
import git
|
|
6
|
+
|
|
7
|
+
from trap.git_ops.url import ParsedGitUrl
|
|
8
|
+
from trap.models.provenance import GitProvenance
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class LocalRepo:
|
|
12
|
+
"""An existing on-disk git checkout — read-only inspection.
|
|
13
|
+
|
|
14
|
+
Distinct from `RemoteRepo` (which clones a declared URL into a root): this wraps
|
|
15
|
+
a `git.Repo` already on disk — whether trap cloned it or the user pointed at
|
|
16
|
+
a local solution. Both clone-sync validation and report provenance go through
|
|
17
|
+
here, so the "open repo + read origin/commit/dirty" logic lives in one place.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
def __init__(self, repo: git.Repo) -> None:
|
|
21
|
+
self.repo = repo
|
|
22
|
+
|
|
23
|
+
@classmethod
|
|
24
|
+
def open(cls, path: Path, *, search_parent: bool = False) -> LocalRepo | None:
|
|
25
|
+
"""Open the git checkout at `path`, or None if it isn't a git repo."""
|
|
26
|
+
try:
|
|
27
|
+
return cls(git.Repo(path, search_parent_directories=search_parent))
|
|
28
|
+
except (git.InvalidGitRepositoryError, git.NoSuchPathError):
|
|
29
|
+
return None
|
|
30
|
+
|
|
31
|
+
@property
|
|
32
|
+
def origin_normalised_url(self) -> str | None:
|
|
33
|
+
"""`origin` remote as a canonical https URL, or None if there's no origin."""
|
|
34
|
+
try:
|
|
35
|
+
return ParsedGitUrl.from_full_url(self.repo.remotes.origin.url).normalised_url
|
|
36
|
+
except AttributeError:
|
|
37
|
+
return None
|
|
38
|
+
|
|
39
|
+
def provenance(self) -> GitProvenance:
|
|
40
|
+
"""{repo, commit} for a clean checkout with an origin, else empty.
|
|
41
|
+
|
|
42
|
+
Empty for a dirty tree (tracked-file changes), a remote-less repo, or any
|
|
43
|
+
git error — the run isn't reproducible from remote+commit alone, so we
|
|
44
|
+
claim nothing. Best-effort: a probe failure degrades to empty rather than
|
|
45
|
+
aborting an otherwise-complete run. Untracked files (run outputs under
|
|
46
|
+
.trap/, .venv, …) don't count as dirty.
|
|
47
|
+
"""
|
|
48
|
+
try:
|
|
49
|
+
url = self.origin_normalised_url
|
|
50
|
+
if url is None or not self.repo.head.is_valid() or self.repo.is_dirty():
|
|
51
|
+
return GitProvenance()
|
|
52
|
+
return GitProvenance(repo=url, commit=self.repo.head.commit.hexsha)
|
|
53
|
+
except Exception:
|
|
54
|
+
return GitProvenance()
|
|
55
|
+
|
|
56
|
+
@classmethod
|
|
57
|
+
def provenance_of(cls, path: Path) -> GitProvenance:
|
|
58
|
+
"""Provenance ({repo, commit}) of the checkout at `path`, empty if not a git repo."""
|
|
59
|
+
local_repo = cls.open(path, search_parent=True)
|
|
60
|
+
return local_repo.provenance() if local_repo else GitProvenance()
|
trap/git_ops/remote.py
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
import git
|
|
6
|
+
|
|
7
|
+
from trap.git_ops.base import GitOpsError, ProgressCallback
|
|
8
|
+
from trap.git_ops.local import LocalRepo
|
|
9
|
+
from trap.git_ops.rev import RevStrategy
|
|
10
|
+
from trap.git_ops.url import ParsedGitUrl
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class RemoteRepo:
|
|
14
|
+
"""A parsed git+ URL bound to a local clone directory (`root`).
|
|
15
|
+
|
|
16
|
+
`ensure()` clones it (fresh) or reconciles an existing clone. The caller
|
|
17
|
+
decides where to clone — `root` is the resolved directory, RemoteRepo holds no
|
|
18
|
+
default or base-dir policy. Takes a `ParsedGitUrl` (callers usually need its
|
|
19
|
+
`basename` to build `root` anyway, so parse once and pass it in).
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
def __init__(self, parsed: ParsedGitUrl, root: Path) -> None:
|
|
23
|
+
self.parsed = parsed
|
|
24
|
+
self.strategy = RevStrategy.for_rev(parsed.rev)
|
|
25
|
+
self.root = root
|
|
26
|
+
|
|
27
|
+
@property
|
|
28
|
+
def local_dir(self) -> Path:
|
|
29
|
+
"""Clone root, descended into the requested subdirectory if any."""
|
|
30
|
+
if self.parsed.subdirectory:
|
|
31
|
+
return self.root / self.parsed.subdirectory
|
|
32
|
+
return self.root
|
|
33
|
+
|
|
34
|
+
def ensure(self, progress_func: ProgressCallback = None) -> bool:
|
|
35
|
+
"""Clone (fresh) or sync (existing) the repo into `root`.
|
|
36
|
+
|
|
37
|
+
Returns True when code changed (fresh clone OR branch fast-forwarded) —
|
|
38
|
+
callers auto-run setup_cmd then. Read `local_dir` for the resolved path.
|
|
39
|
+
"""
|
|
40
|
+
return self._sync(progress_func) if self.root.exists() else self._clone(progress_func)
|
|
41
|
+
|
|
42
|
+
def _clone(self, progress_func: ProgressCallback) -> bool:
|
|
43
|
+
"""Fresh clone — common scaffolding here; rev-specific clone via the strategy."""
|
|
44
|
+
if progress_func:
|
|
45
|
+
progress_func(f"cloning {self.parsed.repo} → {self.root}")
|
|
46
|
+
try:
|
|
47
|
+
self.strategy.clone(self.parsed.repo, self.root)
|
|
48
|
+
except git.GitCommandError as exc:
|
|
49
|
+
raise GitOpsError(f"git clone failed:\n{exc.stderr.strip()}") from exc
|
|
50
|
+
return True
|
|
51
|
+
|
|
52
|
+
def _sync(self, progress_func: ProgressCallback) -> bool:
|
|
53
|
+
"""Existing clone — validate remote here; rev-specific update via the strategy."""
|
|
54
|
+
local_repo = LocalRepo.open(self.root)
|
|
55
|
+
if local_repo is None:
|
|
56
|
+
raise GitOpsError(f"{self.root} is not a git repository")
|
|
57
|
+
|
|
58
|
+
actual = local_repo.origin_normalised_url
|
|
59
|
+
expected = self.parsed.normalised_url
|
|
60
|
+
if actual != expected:
|
|
61
|
+
raise GitOpsError(f"repo mismatch at {self.root}:\n declared: {expected}\n found: {actual}")
|
|
62
|
+
|
|
63
|
+
return self.strategy.reconcile(local_repo.repo, self.root, progress_func)
|