agentdevlabs 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agentdevlabs/__init__.py +7 -0
- agentdevlabs/api_types.py +466 -0
- agentdevlabs/background.py +149 -0
- agentdevlabs/cache.py +113 -0
- agentdevlabs/cache_key.py +69 -0
- agentdevlabs/config.py +95 -0
- agentdevlabs/decrypt.py +18 -0
- agentdevlabs/events.py +93 -0
- agentdevlabs/hook.py +1009 -0
- agentdevlabs/log.py +22 -0
- agentdevlabs/playback.py +173 -0
- agentdevlabs/providers.py +97 -0
- agentdevlabs/recorder.py +226 -0
- agentdevlabs/redact.py +71 -0
- agentdevlabs/replay.py +184 -0
- agentdevlabs/rules.py +55 -0
- agentdevlabs/server.py +446 -0
- agentdevlabs/trace.py +227 -0
- agentdevlabs-0.1.0.dist-info/METADATA +100 -0
- agentdevlabs-0.1.0.dist-info/RECORD +22 -0
- agentdevlabs-0.1.0.dist-info/WHEEL +4 -0
- agentdevlabs-0.1.0.dist-info/licenses/LICENSE +202 -0
agentdevlabs/log.py
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
"""Warnings carry a reason, never a prompt, a response or a credential (SI9)."""
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
from collections.abc import Callable
|
|
5
|
+
|
|
6
|
+
_logger = logging.getLogger("agentdevlabs")
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def warn(message: str) -> None:
|
|
10
|
+
_logger.warning("[agentdevlabs] %s", message)
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def warn_once() -> Callable[[str], None]:
|
|
14
|
+
seen: set[str] = set()
|
|
15
|
+
|
|
16
|
+
def say(message: str) -> None:
|
|
17
|
+
if message in seen:
|
|
18
|
+
return
|
|
19
|
+
seen.add(message)
|
|
20
|
+
warn(message)
|
|
21
|
+
|
|
22
|
+
return say
|
agentdevlabs/playback.py
ADDED
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
import base64
|
|
3
|
+
import json
|
|
4
|
+
import math
|
|
5
|
+
import time
|
|
6
|
+
from collections.abc import AsyncIterator, Callable, Iterator
|
|
7
|
+
from dataclasses import dataclass, field
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
import httpx2
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@dataclass
|
|
14
|
+
class Ending:
|
|
15
|
+
"""How a recording ended: `complete`, `abort`, `error` (a failed answer or stream) or
|
|
16
|
+
`response` (a whole non-streamed answer)."""
|
|
17
|
+
|
|
18
|
+
kind: str = "complete"
|
|
19
|
+
at_ms: int = 0
|
|
20
|
+
status: int = 200
|
|
21
|
+
body: str = ""
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@dataclass
|
|
25
|
+
class Recorded:
|
|
26
|
+
is_stream: bool = False
|
|
27
|
+
ttfb_ms: int = 0
|
|
28
|
+
chunks: list[tuple[int, bytes]] = field(default_factory=list)
|
|
29
|
+
ending: Ending = field(default_factory=Ending)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def parse_document(document: str) -> Recorded:
|
|
33
|
+
"""Reads a decrypted capture document (NDJSON). A malformed one raises and the caller falls
|
|
34
|
+
back to the provider."""
|
|
35
|
+
recorded = Recorded()
|
|
36
|
+
for text in filter(None, document.split("\n")):
|
|
37
|
+
line: dict[str, Any] = json.loads(text)
|
|
38
|
+
kind, at_ms = line["t"], line.get("at_ms") or 0
|
|
39
|
+
if kind == "start":
|
|
40
|
+
recorded.is_stream = line.get("is_stream") is True
|
|
41
|
+
elif kind == "chunk":
|
|
42
|
+
recorded.chunks.append((at_ms, base64.b64decode(line.get("data") or "")))
|
|
43
|
+
elif kind == "abort":
|
|
44
|
+
recorded.ending = Ending("abort", at_ms)
|
|
45
|
+
elif kind in ("error", "response"):
|
|
46
|
+
status, body = line.get("status_code") or 200, line.get("body") or ""
|
|
47
|
+
recorded.ending = Ending(kind, at_ms, status, body)
|
|
48
|
+
elif kind == "end":
|
|
49
|
+
recorded.ttfb_ms = line.get("ttfb_ms") or 0
|
|
50
|
+
return recorded
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _seconds(ms: float, speed: float) -> float:
|
|
54
|
+
"""Waits are divided by `speed`; infinity does not wait."""
|
|
55
|
+
if math.isinf(speed) or ms <= 0:
|
|
56
|
+
return 0.0
|
|
57
|
+
return ms / speed / 1000
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
class _Pacer:
|
|
61
|
+
def __init__(self, recorded: Recorded, speed: float):
|
|
62
|
+
self._recorded = recorded
|
|
63
|
+
self._speed = speed
|
|
64
|
+
self._started = time.perf_counter()
|
|
65
|
+
|
|
66
|
+
def due_in(self, at_ms: int) -> float:
|
|
67
|
+
wait = _seconds(self._recorded.ttfb_ms + at_ms, self._speed)
|
|
68
|
+
return wait - (time.perf_counter() - self._started)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _whole_response(recorded: Recorded, request: httpx2.Request) -> httpx2.Response:
|
|
72
|
+
ending = recorded.ending
|
|
73
|
+
has_body = ending.kind in ("response", "error")
|
|
74
|
+
return httpx2.Response(
|
|
75
|
+
ending.status if has_body else 200,
|
|
76
|
+
headers={"content-type": "application/json"},
|
|
77
|
+
content=ending.body.encode("utf-8") if has_body else b"",
|
|
78
|
+
request=request,
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def _stream_error(ending: Ending) -> httpx2.HTTPError:
|
|
83
|
+
if ending.kind == "abort":
|
|
84
|
+
return httpx2.ReadError("The recorded stream was aborted")
|
|
85
|
+
return httpx2.ReadError(ending.body)
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _wait_for_whole(recorded: Recorded, speed: float) -> float:
|
|
89
|
+
ends_at = 0 if recorded.ending.kind == "complete" else recorded.ending.at_ms
|
|
90
|
+
return _seconds(recorded.ttfb_ms + ends_at, speed)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
class _OnceEnd:
|
|
94
|
+
def __init__(self, on_end: Callable[[], None]):
|
|
95
|
+
self._on_end = on_end
|
|
96
|
+
self._done = False
|
|
97
|
+
|
|
98
|
+
def __call__(self) -> None:
|
|
99
|
+
if not self._done:
|
|
100
|
+
self._done = True
|
|
101
|
+
self._on_end()
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
class PlayedStream(httpx2.SyncByteStream):
|
|
105
|
+
def __init__(self, recorded: Recorded, speed: float, on_end: Callable[[], None]):
|
|
106
|
+
self._recorded = recorded
|
|
107
|
+
self._pacer = _Pacer(recorded, speed)
|
|
108
|
+
self._end = _OnceEnd(on_end)
|
|
109
|
+
|
|
110
|
+
def __iter__(self) -> Iterator[bytes]:
|
|
111
|
+
try:
|
|
112
|
+
for at_ms, data in self._recorded.chunks:
|
|
113
|
+
time.sleep(max(0.0, self._pacer.due_in(at_ms)))
|
|
114
|
+
yield data
|
|
115
|
+
ending = self._recorded.ending
|
|
116
|
+
if ending.kind != "complete":
|
|
117
|
+
time.sleep(max(0.0, self._pacer.due_in(ending.at_ms)))
|
|
118
|
+
raise _stream_error(ending)
|
|
119
|
+
finally:
|
|
120
|
+
self._end()
|
|
121
|
+
|
|
122
|
+
def close(self) -> None:
|
|
123
|
+
self._end()
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
class PlayedAsyncStream(httpx2.AsyncByteStream):
|
|
127
|
+
def __init__(self, recorded: Recorded, speed: float, on_end: Callable[[], None]):
|
|
128
|
+
self._recorded = recorded
|
|
129
|
+
self._pacer = _Pacer(recorded, speed)
|
|
130
|
+
self._end = _OnceEnd(on_end)
|
|
131
|
+
|
|
132
|
+
async def __aiter__(self) -> AsyncIterator[bytes]:
|
|
133
|
+
try:
|
|
134
|
+
for at_ms, data in self._recorded.chunks:
|
|
135
|
+
await asyncio.sleep(max(0.0, self._pacer.due_in(at_ms)))
|
|
136
|
+
yield data
|
|
137
|
+
ending = self._recorded.ending
|
|
138
|
+
if ending.kind != "complete":
|
|
139
|
+
await asyncio.sleep(max(0.0, self._pacer.due_in(ending.at_ms)))
|
|
140
|
+
raise _stream_error(ending)
|
|
141
|
+
finally:
|
|
142
|
+
self._end()
|
|
143
|
+
|
|
144
|
+
async def aclose(self) -> None:
|
|
145
|
+
self._end()
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def _streamed_response(stream: httpx2.SyncByteStream | httpx2.AsyncByteStream, request):
|
|
149
|
+
return httpx2.Response(
|
|
150
|
+
200, headers={"content-type": "text/event-stream"}, stream=stream, request=request
|
|
151
|
+
)
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def playback(
|
|
155
|
+
recorded: Recorded, speed: float, on_end: Callable[[], None], request: httpx2.Request
|
|
156
|
+
) -> httpx2.Response:
|
|
157
|
+
"""Plays a recorded response back (F1, F2, F3). `on_end` runs once, when the body closes,
|
|
158
|
+
errors or is closed early."""
|
|
159
|
+
if recorded.is_stream:
|
|
160
|
+
return _streamed_response(PlayedStream(recorded, speed, on_end), request)
|
|
161
|
+
time.sleep(_wait_for_whole(recorded, speed))
|
|
162
|
+
on_end()
|
|
163
|
+
return _whole_response(recorded, request)
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
async def aplayback(
|
|
167
|
+
recorded: Recorded, speed: float, on_end: Callable[[], None], request: httpx2.Request
|
|
168
|
+
) -> httpx2.Response:
|
|
169
|
+
if recorded.is_stream:
|
|
170
|
+
return _streamed_response(PlayedAsyncStream(recorded, speed, on_end), request)
|
|
171
|
+
await asyncio.sleep(_wait_for_whole(recorded, speed))
|
|
172
|
+
on_end()
|
|
173
|
+
return _whole_response(recorded, request)
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
import json
|
|
2
|
+
from typing import Any
|
|
3
|
+
from urllib.parse import urlsplit
|
|
4
|
+
|
|
5
|
+
import httpx2
|
|
6
|
+
|
|
7
|
+
# The endpoints we record. Everything else passes through.
|
|
8
|
+
ENDPOINTS = (
|
|
9
|
+
("/chat/completions", "openai"),
|
|
10
|
+
("/responses", "openai"),
|
|
11
|
+
("/v1/messages", "anthropic"),
|
|
12
|
+
)
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def provider_of(url: str, method: str) -> str | None:
|
|
16
|
+
if method != "POST":
|
|
17
|
+
return None
|
|
18
|
+
path = urlsplit(url).path
|
|
19
|
+
return next((provider for suffix, provider in ENDPOINTS if path.endswith(suffix)), None)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def will_be_retried(response: httpx2.Response) -> bool:
|
|
23
|
+
"""The provider SDKs resend these statuses (and any response that says `x-should-retry:
|
|
24
|
+
true`), so the app may never see the first answer."""
|
|
25
|
+
verdict = response.headers.get("x-should-retry")
|
|
26
|
+
if verdict:
|
|
27
|
+
return verdict == "true"
|
|
28
|
+
return response.status_code in (408, 409, 429) or response.status_code >= 500
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def is_retry(headers: httpx2.Headers) -> bool:
|
|
32
|
+
try:
|
|
33
|
+
return int(headers.get("x-stainless-retry-count", "0")) > 0
|
|
34
|
+
except ValueError:
|
|
35
|
+
return False
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _last_turn(body: dict[str, Any]) -> dict[str, Any] | None:
|
|
39
|
+
turns = body.get("messages", body.get("input"))
|
|
40
|
+
if isinstance(turns, list) and turns and isinstance(turns[-1], dict):
|
|
41
|
+
return turns[-1]
|
|
42
|
+
return None
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def ends_with_tool_result(body: dict[str, Any]) -> bool:
|
|
46
|
+
"""S3: a final user text message starts a trace; a final tool result continues the one
|
|
47
|
+
before."""
|
|
48
|
+
turn = _last_turn(body)
|
|
49
|
+
if turn is None:
|
|
50
|
+
return False
|
|
51
|
+
if turn.get("role") == "tool" or turn.get("type") == "function_call_output":
|
|
52
|
+
return True
|
|
53
|
+
content = turn.get("content")
|
|
54
|
+
return isinstance(content, list) and any(
|
|
55
|
+
isinstance(part, dict) and part.get("type") == "tool_result" for part in content
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def usage_in(events: list[Any]) -> tuple[int | None, int | None]:
|
|
60
|
+
"""Token counts sit under `usage` (or `message.usage`, `response.usage`) in every shape we
|
|
61
|
+
record; a later event's count replaces an earlier one's."""
|
|
62
|
+
input_tokens = output_tokens = None
|
|
63
|
+
for event in events:
|
|
64
|
+
if not isinstance(event, dict):
|
|
65
|
+
continue
|
|
66
|
+
found = _usage_of(event)
|
|
67
|
+
if found is None:
|
|
68
|
+
continue
|
|
69
|
+
input_tokens = _first_present(found, "prompt_tokens", "input_tokens", input_tokens)
|
|
70
|
+
output_tokens = _first_present(found, "completion_tokens", "output_tokens", output_tokens)
|
|
71
|
+
return input_tokens, output_tokens
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _usage_of(event: dict[str, Any]) -> dict[str, Any] | None:
|
|
75
|
+
for holder in (event, event.get("message"), event.get("response")):
|
|
76
|
+
if isinstance(holder, dict) and isinstance(holder.get("usage"), dict):
|
|
77
|
+
return holder["usage"]
|
|
78
|
+
return None
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _first_present(counts: dict[str, Any], first: str, second: str, fallback: int | None):
|
|
82
|
+
for name in (first, second):
|
|
83
|
+
if counts.get(name) is not None:
|
|
84
|
+
return counts[name]
|
|
85
|
+
return fallback
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def events_in(text: str) -> list[Any]:
|
|
89
|
+
events = []
|
|
90
|
+
for line in text.split("\n"):
|
|
91
|
+
if not line.startswith("data:") or "[DONE]" in line:
|
|
92
|
+
continue
|
|
93
|
+
try:
|
|
94
|
+
events.append(json.loads(line[5:]))
|
|
95
|
+
except ValueError:
|
|
96
|
+
continue
|
|
97
|
+
return events
|
agentdevlabs/recorder.py
ADDED
|
@@ -0,0 +1,226 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
import codecs
|
|
3
|
+
import json
|
|
4
|
+
import re
|
|
5
|
+
import time
|
|
6
|
+
from base64 import b64encode
|
|
7
|
+
from collections.abc import AsyncIterable, AsyncIterator, Callable, Iterable, Iterator
|
|
8
|
+
from dataclasses import dataclass, field
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
from .log import warn
|
|
12
|
+
from .providers import events_in, usage_in
|
|
13
|
+
from .redact import redact
|
|
14
|
+
from .trace import Trace
|
|
15
|
+
|
|
16
|
+
Deliver = Callable[[str | None], None]
|
|
17
|
+
DONE = re.compile(r"data: \[DONE\]\s*$")
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@dataclass
|
|
21
|
+
class Call:
|
|
22
|
+
trace: Trace
|
|
23
|
+
step: int
|
|
24
|
+
cache_key: str
|
|
25
|
+
provider: str
|
|
26
|
+
model: str
|
|
27
|
+
request: Any
|
|
28
|
+
group: str | None = None
|
|
29
|
+
rules: dict[str, Any] | None = None
|
|
30
|
+
started_at: float = field(default_factory=time.perf_counter)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _ms(seconds: float) -> int:
|
|
34
|
+
return round(seconds * 1000)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class Recording:
|
|
38
|
+
"""Builds one capture document (NDJSON) as the response passes by, and hands it over once the
|
|
39
|
+
call ends. `deliver` always runs once: with the document, or with None when the call is not
|
|
40
|
+
recorded."""
|
|
41
|
+
|
|
42
|
+
def __init__(self, call: Call, status: int, is_stream: bool, deliver: Deliver):
|
|
43
|
+
self._call = call
|
|
44
|
+
self._status = status
|
|
45
|
+
self._is_stream = is_stream
|
|
46
|
+
self._deliver = deliver
|
|
47
|
+
self._lines: list[dict[str, Any]] = []
|
|
48
|
+
self._text: list[str] = []
|
|
49
|
+
self._decoder = codecs.getincrementaldecoder("utf-8")(errors="replace")
|
|
50
|
+
self._first_byte_at: float | None = None
|
|
51
|
+
self._finished = False
|
|
52
|
+
|
|
53
|
+
def guard(self, step: Callable[[], None]) -> None:
|
|
54
|
+
"""Our copy of the response must never break the app's: a failure here drops the
|
|
55
|
+
recording and warns."""
|
|
56
|
+
try:
|
|
57
|
+
step()
|
|
58
|
+
except Exception:
|
|
59
|
+
self.discard()
|
|
60
|
+
warn("a response could not be recorded, so this call is not captured")
|
|
61
|
+
|
|
62
|
+
def discard(self) -> None:
|
|
63
|
+
if self._finished:
|
|
64
|
+
return
|
|
65
|
+
self._finished = True
|
|
66
|
+
self._deliver(None)
|
|
67
|
+
|
|
68
|
+
def chunk(self, data: bytes, arrived: float) -> None:
|
|
69
|
+
if self._first_byte_at is None:
|
|
70
|
+
self._first_byte_at = arrived
|
|
71
|
+
self._text.append(self._decoder.decode(data))
|
|
72
|
+
if self._is_stream:
|
|
73
|
+
self._lines.append(
|
|
74
|
+
{
|
|
75
|
+
"t": "chunk",
|
|
76
|
+
"at_ms": self._since_first_byte(arrived),
|
|
77
|
+
"data": b64encode(data).decode("ascii"),
|
|
78
|
+
}
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
def cancel(self) -> None:
|
|
82
|
+
"""The OpenAI client closes the body once it has read `[DONE]`: the stream was complete,
|
|
83
|
+
not abandoned."""
|
|
84
|
+
if DONE.search("".join(self._text)):
|
|
85
|
+
self.complete()
|
|
86
|
+
else:
|
|
87
|
+
self.abort()
|
|
88
|
+
|
|
89
|
+
def complete(self) -> None:
|
|
90
|
+
body = self._body_text()
|
|
91
|
+
if not 200 <= self._status < 300:
|
|
92
|
+
self._lines.append(self._body_line("error", body))
|
|
93
|
+
self._finish("error", [])
|
|
94
|
+
return
|
|
95
|
+
if not self._is_stream:
|
|
96
|
+
self._lines.append(self._body_line("response", body))
|
|
97
|
+
events = events_in(body) if self._is_stream else [_safe_json(body)]
|
|
98
|
+
self._finish("complete", events)
|
|
99
|
+
|
|
100
|
+
def abort(self) -> None:
|
|
101
|
+
self._lines.append({"t": "abort", "at_ms": self._since_first_byte()})
|
|
102
|
+
self._finish("aborted", events_in(self._body_text()))
|
|
103
|
+
|
|
104
|
+
def fail(self, error: BaseException) -> None:
|
|
105
|
+
self._lines.append(self._body_line("error", str(error) or "the response ended early"))
|
|
106
|
+
self._finish("error", events_in(self._body_text()))
|
|
107
|
+
|
|
108
|
+
def _body_line(self, kind: str, body: str) -> dict[str, Any]:
|
|
109
|
+
return {
|
|
110
|
+
"t": kind,
|
|
111
|
+
"at_ms": self._since_first_byte(),
|
|
112
|
+
"status_code": self._status,
|
|
113
|
+
"body": body,
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
# Offsets in the document count from the first byte, so replay can pace itself without the
|
|
117
|
+
# provider's wait.
|
|
118
|
+
def _since_first_byte(self, at: float | None = None) -> int:
|
|
119
|
+
if self._first_byte_at is None:
|
|
120
|
+
return 0
|
|
121
|
+
return _ms((time.perf_counter() if at is None else at) - self._first_byte_at)
|
|
122
|
+
|
|
123
|
+
def _body_text(self) -> str:
|
|
124
|
+
self._text.append(self._decoder.decode(b"", final=True))
|
|
125
|
+
return "".join(self._text)
|
|
126
|
+
|
|
127
|
+
def _finish(self, status: str, events: list[Any]) -> None:
|
|
128
|
+
if self._finished:
|
|
129
|
+
return
|
|
130
|
+
# Marked finished only once the document is built, so a failure here still reaches
|
|
131
|
+
# `discard`.
|
|
132
|
+
document = self._document(status, events)
|
|
133
|
+
self._finished = True
|
|
134
|
+
self._deliver(document)
|
|
135
|
+
|
|
136
|
+
def _document(self, status: str, events: list[Any]) -> str:
|
|
137
|
+
call = self._call
|
|
138
|
+
input_tokens, output_tokens = usage_in(events)
|
|
139
|
+
start = {
|
|
140
|
+
"t": "start",
|
|
141
|
+
"v": 1,
|
|
142
|
+
"trace_id": call.trace.id,
|
|
143
|
+
"first_cache_key": call.trace.first_cache_key,
|
|
144
|
+
"step": call.step,
|
|
145
|
+
"cache_key": call.cache_key,
|
|
146
|
+
"provider": call.provider,
|
|
147
|
+
"model": call.model,
|
|
148
|
+
"is_stream": self._is_stream,
|
|
149
|
+
"request": redact(call.request, call.rules),
|
|
150
|
+
**({"group": call.group} if call.group else {}),
|
|
151
|
+
}
|
|
152
|
+
end = {
|
|
153
|
+
"t": "end",
|
|
154
|
+
"status": status,
|
|
155
|
+
"ttfb_ms": (
|
|
156
|
+
None if self._first_byte_at is None else _ms(self._first_byte_at - call.started_at)
|
|
157
|
+
),
|
|
158
|
+
"duration_ms": _ms(time.perf_counter() - call.started_at),
|
|
159
|
+
"input_tokens": input_tokens,
|
|
160
|
+
"output_tokens": output_tokens,
|
|
161
|
+
}
|
|
162
|
+
lines = [start, *self._lines, end]
|
|
163
|
+
return "\n".join(json.dumps(line, separators=(",", ":")) for line in lines) + "\n"
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def _safe_json(text: str) -> Any:
|
|
167
|
+
try:
|
|
168
|
+
return json.loads(text)
|
|
169
|
+
except ValueError:
|
|
170
|
+
return None
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def record_stream(
|
|
174
|
+
call: Call,
|
|
175
|
+
inner: Iterable[bytes],
|
|
176
|
+
deliver: Deliver,
|
|
177
|
+
status: int = 200,
|
|
178
|
+
is_stream: bool = True,
|
|
179
|
+
) -> Iterator[bytes]:
|
|
180
|
+
"""The app gets chunks as the provider sends them; the app leaving early shows up as the
|
|
181
|
+
generator being closed, which is an abort."""
|
|
182
|
+
recording = Recording(call, status, is_stream, deliver)
|
|
183
|
+
try:
|
|
184
|
+
for chunk in inner:
|
|
185
|
+
arrived = time.perf_counter()
|
|
186
|
+
try:
|
|
187
|
+
yield chunk
|
|
188
|
+
finally:
|
|
189
|
+
# Recorded once the app has the chunk (F4), at the time it arrived.
|
|
190
|
+
recording.guard(
|
|
191
|
+
lambda chunk=chunk, arrived=arrived: recording.chunk(chunk, arrived)
|
|
192
|
+
)
|
|
193
|
+
except GeneratorExit:
|
|
194
|
+
recording.guard(recording.cancel)
|
|
195
|
+
raise
|
|
196
|
+
except Exception as error:
|
|
197
|
+
recording.guard(lambda error=error: recording.fail(error))
|
|
198
|
+
raise
|
|
199
|
+
recording.guard(recording.complete)
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
async def record_astream(
|
|
203
|
+
call: Call,
|
|
204
|
+
inner: AsyncIterable[bytes],
|
|
205
|
+
deliver: Deliver,
|
|
206
|
+
status: int = 200,
|
|
207
|
+
is_stream: bool = True,
|
|
208
|
+
) -> AsyncIterator[bytes]:
|
|
209
|
+
recording = Recording(call, status, is_stream, deliver)
|
|
210
|
+
try:
|
|
211
|
+
async for chunk in inner:
|
|
212
|
+
arrived = time.perf_counter()
|
|
213
|
+
try:
|
|
214
|
+
yield chunk
|
|
215
|
+
finally:
|
|
216
|
+
# Recorded once the app has the chunk (F4), at the time it arrived.
|
|
217
|
+
recording.guard(
|
|
218
|
+
lambda chunk=chunk, arrived=arrived: recording.chunk(chunk, arrived)
|
|
219
|
+
)
|
|
220
|
+
except (GeneratorExit, asyncio.CancelledError):
|
|
221
|
+
recording.guard(recording.cancel)
|
|
222
|
+
raise
|
|
223
|
+
except Exception as error:
|
|
224
|
+
recording.guard(lambda error=error: recording.fail(error))
|
|
225
|
+
raise
|
|
226
|
+
recording.guard(recording.complete)
|
agentdevlabs/redact.py
ADDED
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
"""SI3: the upload copy of a request loses credential fields and anything shaped like a secret.
|
|
2
|
+
|
|
3
|
+
The patterns are spec/redaction/README.md's table, in its order. The cache key is hashed before this
|
|
4
|
+
runs, and responses never pass through it.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import re
|
|
8
|
+
from functools import reduce
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
REDACTED = "[redacted]"
|
|
12
|
+
|
|
13
|
+
CREDENTIAL_PATHS = [
|
|
14
|
+
path.split(".")
|
|
15
|
+
for path in (
|
|
16
|
+
"tools.*.authorization",
|
|
17
|
+
"tools.*.headers.*",
|
|
18
|
+
"mcp_servers.*.authorization_token",
|
|
19
|
+
)
|
|
20
|
+
]
|
|
21
|
+
|
|
22
|
+
PATTERNS = [
|
|
23
|
+
re.compile(pattern)
|
|
24
|
+
for pattern in (
|
|
25
|
+
r"-----BEGIN [A-Z ]*PRIVATE KEY( BLOCK)?-----"
|
|
26
|
+
r"([^-]|-[^-])*"
|
|
27
|
+
r"-----END [A-Z ]*PRIVATE KEY( BLOCK)?-----",
|
|
28
|
+
r"adl_(pat|svc)_[A-Za-z0-9]{40}",
|
|
29
|
+
r"sk-(ant|proj|svcacct|admin|None)-[A-Za-z0-9_-]{40,}",
|
|
30
|
+
r"sk-[A-Za-z0-9]{40,}",
|
|
31
|
+
r"[Bb]earer [A-Za-z0-9._~+/=-]{20,}",
|
|
32
|
+
r"(AKIA|ASIA)[A-Z0-9]{16}",
|
|
33
|
+
r"gh[pousr]_[A-Za-z0-9]{36,}",
|
|
34
|
+
r"github_pat_[A-Za-z0-9_]{50,}",
|
|
35
|
+
r"AIza[A-Za-z0-9_-]{35}",
|
|
36
|
+
r"xox[abprs]-[A-Za-z0-9-]{10,}",
|
|
37
|
+
)
|
|
38
|
+
]
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _redact_path(value: Any, path: list[str]) -> Any:
|
|
42
|
+
if not isinstance(value, (dict, list)):
|
|
43
|
+
return value
|
|
44
|
+
head, rest = path[0], path[1:]
|
|
45
|
+
items = value.items() if isinstance(value, dict) else enumerate(value)
|
|
46
|
+
copied = dict(items) if isinstance(value, dict) else dict(enumerate(value))
|
|
47
|
+
for key in copied:
|
|
48
|
+
if head == "*" or str(key) == head:
|
|
49
|
+
copied[key] = REDACTED if not rest else _redact_path(copied[key], rest)
|
|
50
|
+
return copied if isinstance(value, dict) else list(copied.values())
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _redact_strings(value: Any, patterns: list[re.Pattern]) -> Any:
|
|
54
|
+
if isinstance(value, str):
|
|
55
|
+
return reduce(lambda text, pattern: pattern.sub(REDACTED, text), patterns, value)
|
|
56
|
+
if isinstance(value, list):
|
|
57
|
+
return [_redact_strings(item, patterns) for item in value]
|
|
58
|
+
if isinstance(value, dict):
|
|
59
|
+
return {key: _redact_strings(member, patterns) for key, member in value.items()}
|
|
60
|
+
return value
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def redact(request: Any, rules: dict[str, list[str]] | None = None) -> Any:
|
|
64
|
+
"""A copy of `request` that is safe to upload.
|
|
65
|
+
|
|
66
|
+
Order: credential fields, the project's fields, the built-in patterns, the project's patterns.
|
|
67
|
+
"""
|
|
68
|
+
rules = rules or {}
|
|
69
|
+
paths = CREDENTIAL_PATHS + [path.split(".") for path in rules.get("redact_fields", [])]
|
|
70
|
+
project_patterns = [re.compile(pattern) for pattern in rules.get("redact_patterns", [])]
|
|
71
|
+
return _redact_strings(reduce(_redact_path, paths, request), PATTERNS + project_patterns)
|