agentdevlabs 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
agentdevlabs/log.py ADDED
@@ -0,0 +1,22 @@
1
+ """Warnings carry a reason, never a prompt, a response or a credential (SI9)."""
2
+
3
+ import logging
4
+ from collections.abc import Callable
5
+
6
+ _logger = logging.getLogger("agentdevlabs")
7
+
8
+
9
+ def warn(message: str) -> None:
10
+ _logger.warning("[agentdevlabs] %s", message)
11
+
12
+
13
+ def warn_once() -> Callable[[str], None]:
14
+ seen: set[str] = set()
15
+
16
+ def say(message: str) -> None:
17
+ if message in seen:
18
+ return
19
+ seen.add(message)
20
+ warn(message)
21
+
22
+ return say
@@ -0,0 +1,173 @@
1
+ import asyncio
2
+ import base64
3
+ import json
4
+ import math
5
+ import time
6
+ from collections.abc import AsyncIterator, Callable, Iterator
7
+ from dataclasses import dataclass, field
8
+ from typing import Any
9
+
10
+ import httpx2
11
+
12
+
13
+ @dataclass
14
+ class Ending:
15
+ """How a recording ended: `complete`, `abort`, `error` (a failed answer or stream) or
16
+ `response` (a whole non-streamed answer)."""
17
+
18
+ kind: str = "complete"
19
+ at_ms: int = 0
20
+ status: int = 200
21
+ body: str = ""
22
+
23
+
24
+ @dataclass
25
+ class Recorded:
26
+ is_stream: bool = False
27
+ ttfb_ms: int = 0
28
+ chunks: list[tuple[int, bytes]] = field(default_factory=list)
29
+ ending: Ending = field(default_factory=Ending)
30
+
31
+
32
+ def parse_document(document: str) -> Recorded:
33
+ """Reads a decrypted capture document (NDJSON). A malformed one raises and the caller falls
34
+ back to the provider."""
35
+ recorded = Recorded()
36
+ for text in filter(None, document.split("\n")):
37
+ line: dict[str, Any] = json.loads(text)
38
+ kind, at_ms = line["t"], line.get("at_ms") or 0
39
+ if kind == "start":
40
+ recorded.is_stream = line.get("is_stream") is True
41
+ elif kind == "chunk":
42
+ recorded.chunks.append((at_ms, base64.b64decode(line.get("data") or "")))
43
+ elif kind == "abort":
44
+ recorded.ending = Ending("abort", at_ms)
45
+ elif kind in ("error", "response"):
46
+ status, body = line.get("status_code") or 200, line.get("body") or ""
47
+ recorded.ending = Ending(kind, at_ms, status, body)
48
+ elif kind == "end":
49
+ recorded.ttfb_ms = line.get("ttfb_ms") or 0
50
+ return recorded
51
+
52
+
53
+ def _seconds(ms: float, speed: float) -> float:
54
+ """Waits are divided by `speed`; infinity does not wait."""
55
+ if math.isinf(speed) or ms <= 0:
56
+ return 0.0
57
+ return ms / speed / 1000
58
+
59
+
60
+ class _Pacer:
61
+ def __init__(self, recorded: Recorded, speed: float):
62
+ self._recorded = recorded
63
+ self._speed = speed
64
+ self._started = time.perf_counter()
65
+
66
+ def due_in(self, at_ms: int) -> float:
67
+ wait = _seconds(self._recorded.ttfb_ms + at_ms, self._speed)
68
+ return wait - (time.perf_counter() - self._started)
69
+
70
+
71
+ def _whole_response(recorded: Recorded, request: httpx2.Request) -> httpx2.Response:
72
+ ending = recorded.ending
73
+ has_body = ending.kind in ("response", "error")
74
+ return httpx2.Response(
75
+ ending.status if has_body else 200,
76
+ headers={"content-type": "application/json"},
77
+ content=ending.body.encode("utf-8") if has_body else b"",
78
+ request=request,
79
+ )
80
+
81
+
82
+ def _stream_error(ending: Ending) -> httpx2.HTTPError:
83
+ if ending.kind == "abort":
84
+ return httpx2.ReadError("The recorded stream was aborted")
85
+ return httpx2.ReadError(ending.body)
86
+
87
+
88
+ def _wait_for_whole(recorded: Recorded, speed: float) -> float:
89
+ ends_at = 0 if recorded.ending.kind == "complete" else recorded.ending.at_ms
90
+ return _seconds(recorded.ttfb_ms + ends_at, speed)
91
+
92
+
93
+ class _OnceEnd:
94
+ def __init__(self, on_end: Callable[[], None]):
95
+ self._on_end = on_end
96
+ self._done = False
97
+
98
+ def __call__(self) -> None:
99
+ if not self._done:
100
+ self._done = True
101
+ self._on_end()
102
+
103
+
104
+ class PlayedStream(httpx2.SyncByteStream):
105
+ def __init__(self, recorded: Recorded, speed: float, on_end: Callable[[], None]):
106
+ self._recorded = recorded
107
+ self._pacer = _Pacer(recorded, speed)
108
+ self._end = _OnceEnd(on_end)
109
+
110
+ def __iter__(self) -> Iterator[bytes]:
111
+ try:
112
+ for at_ms, data in self._recorded.chunks:
113
+ time.sleep(max(0.0, self._pacer.due_in(at_ms)))
114
+ yield data
115
+ ending = self._recorded.ending
116
+ if ending.kind != "complete":
117
+ time.sleep(max(0.0, self._pacer.due_in(ending.at_ms)))
118
+ raise _stream_error(ending)
119
+ finally:
120
+ self._end()
121
+
122
+ def close(self) -> None:
123
+ self._end()
124
+
125
+
126
+ class PlayedAsyncStream(httpx2.AsyncByteStream):
127
+ def __init__(self, recorded: Recorded, speed: float, on_end: Callable[[], None]):
128
+ self._recorded = recorded
129
+ self._pacer = _Pacer(recorded, speed)
130
+ self._end = _OnceEnd(on_end)
131
+
132
+ async def __aiter__(self) -> AsyncIterator[bytes]:
133
+ try:
134
+ for at_ms, data in self._recorded.chunks:
135
+ await asyncio.sleep(max(0.0, self._pacer.due_in(at_ms)))
136
+ yield data
137
+ ending = self._recorded.ending
138
+ if ending.kind != "complete":
139
+ await asyncio.sleep(max(0.0, self._pacer.due_in(ending.at_ms)))
140
+ raise _stream_error(ending)
141
+ finally:
142
+ self._end()
143
+
144
+ async def aclose(self) -> None:
145
+ self._end()
146
+
147
+
148
+ def _streamed_response(stream: httpx2.SyncByteStream | httpx2.AsyncByteStream, request):
149
+ return httpx2.Response(
150
+ 200, headers={"content-type": "text/event-stream"}, stream=stream, request=request
151
+ )
152
+
153
+
154
+ def playback(
155
+ recorded: Recorded, speed: float, on_end: Callable[[], None], request: httpx2.Request
156
+ ) -> httpx2.Response:
157
+ """Plays a recorded response back (F1, F2, F3). `on_end` runs once, when the body closes,
158
+ errors or is closed early."""
159
+ if recorded.is_stream:
160
+ return _streamed_response(PlayedStream(recorded, speed, on_end), request)
161
+ time.sleep(_wait_for_whole(recorded, speed))
162
+ on_end()
163
+ return _whole_response(recorded, request)
164
+
165
+
166
+ async def aplayback(
167
+ recorded: Recorded, speed: float, on_end: Callable[[], None], request: httpx2.Request
168
+ ) -> httpx2.Response:
169
+ if recorded.is_stream:
170
+ return _streamed_response(PlayedAsyncStream(recorded, speed, on_end), request)
171
+ await asyncio.sleep(_wait_for_whole(recorded, speed))
172
+ on_end()
173
+ return _whole_response(recorded, request)
@@ -0,0 +1,97 @@
1
+ import json
2
+ from typing import Any
3
+ from urllib.parse import urlsplit
4
+
5
+ import httpx2
6
+
7
+ # The endpoints we record. Everything else passes through.
8
+ ENDPOINTS = (
9
+ ("/chat/completions", "openai"),
10
+ ("/responses", "openai"),
11
+ ("/v1/messages", "anthropic"),
12
+ )
13
+
14
+
15
+ def provider_of(url: str, method: str) -> str | None:
16
+ if method != "POST":
17
+ return None
18
+ path = urlsplit(url).path
19
+ return next((provider for suffix, provider in ENDPOINTS if path.endswith(suffix)), None)
20
+
21
+
22
+ def will_be_retried(response: httpx2.Response) -> bool:
23
+ """The provider SDKs resend these statuses (and any response that says `x-should-retry:
24
+ true`), so the app may never see the first answer."""
25
+ verdict = response.headers.get("x-should-retry")
26
+ if verdict:
27
+ return verdict == "true"
28
+ return response.status_code in (408, 409, 429) or response.status_code >= 500
29
+
30
+
31
+ def is_retry(headers: httpx2.Headers) -> bool:
32
+ try:
33
+ return int(headers.get("x-stainless-retry-count", "0")) > 0
34
+ except ValueError:
35
+ return False
36
+
37
+
38
+ def _last_turn(body: dict[str, Any]) -> dict[str, Any] | None:
39
+ turns = body.get("messages", body.get("input"))
40
+ if isinstance(turns, list) and turns and isinstance(turns[-1], dict):
41
+ return turns[-1]
42
+ return None
43
+
44
+
45
+ def ends_with_tool_result(body: dict[str, Any]) -> bool:
46
+ """S3: a final user text message starts a trace; a final tool result continues the one
47
+ before."""
48
+ turn = _last_turn(body)
49
+ if turn is None:
50
+ return False
51
+ if turn.get("role") == "tool" or turn.get("type") == "function_call_output":
52
+ return True
53
+ content = turn.get("content")
54
+ return isinstance(content, list) and any(
55
+ isinstance(part, dict) and part.get("type") == "tool_result" for part in content
56
+ )
57
+
58
+
59
+ def usage_in(events: list[Any]) -> tuple[int | None, int | None]:
60
+ """Token counts sit under `usage` (or `message.usage`, `response.usage`) in every shape we
61
+ record; a later event's count replaces an earlier one's."""
62
+ input_tokens = output_tokens = None
63
+ for event in events:
64
+ if not isinstance(event, dict):
65
+ continue
66
+ found = _usage_of(event)
67
+ if found is None:
68
+ continue
69
+ input_tokens = _first_present(found, "prompt_tokens", "input_tokens", input_tokens)
70
+ output_tokens = _first_present(found, "completion_tokens", "output_tokens", output_tokens)
71
+ return input_tokens, output_tokens
72
+
73
+
74
+ def _usage_of(event: dict[str, Any]) -> dict[str, Any] | None:
75
+ for holder in (event, event.get("message"), event.get("response")):
76
+ if isinstance(holder, dict) and isinstance(holder.get("usage"), dict):
77
+ return holder["usage"]
78
+ return None
79
+
80
+
81
+ def _first_present(counts: dict[str, Any], first: str, second: str, fallback: int | None):
82
+ for name in (first, second):
83
+ if counts.get(name) is not None:
84
+ return counts[name]
85
+ return fallback
86
+
87
+
88
+ def events_in(text: str) -> list[Any]:
89
+ events = []
90
+ for line in text.split("\n"):
91
+ if not line.startswith("data:") or "[DONE]" in line:
92
+ continue
93
+ try:
94
+ events.append(json.loads(line[5:]))
95
+ except ValueError:
96
+ continue
97
+ return events
@@ -0,0 +1,226 @@
1
+ import asyncio
2
+ import codecs
3
+ import json
4
+ import re
5
+ import time
6
+ from base64 import b64encode
7
+ from collections.abc import AsyncIterable, AsyncIterator, Callable, Iterable, Iterator
8
+ from dataclasses import dataclass, field
9
+ from typing import Any
10
+
11
+ from .log import warn
12
+ from .providers import events_in, usage_in
13
+ from .redact import redact
14
+ from .trace import Trace
15
+
16
+ Deliver = Callable[[str | None], None]
17
+ DONE = re.compile(r"data: \[DONE\]\s*$")
18
+
19
+
20
+ @dataclass
21
+ class Call:
22
+ trace: Trace
23
+ step: int
24
+ cache_key: str
25
+ provider: str
26
+ model: str
27
+ request: Any
28
+ group: str | None = None
29
+ rules: dict[str, Any] | None = None
30
+ started_at: float = field(default_factory=time.perf_counter)
31
+
32
+
33
+ def _ms(seconds: float) -> int:
34
+ return round(seconds * 1000)
35
+
36
+
37
+ class Recording:
38
+ """Builds one capture document (NDJSON) as the response passes by, and hands it over once the
39
+ call ends. `deliver` always runs once: with the document, or with None when the call is not
40
+ recorded."""
41
+
42
+ def __init__(self, call: Call, status: int, is_stream: bool, deliver: Deliver):
43
+ self._call = call
44
+ self._status = status
45
+ self._is_stream = is_stream
46
+ self._deliver = deliver
47
+ self._lines: list[dict[str, Any]] = []
48
+ self._text: list[str] = []
49
+ self._decoder = codecs.getincrementaldecoder("utf-8")(errors="replace")
50
+ self._first_byte_at: float | None = None
51
+ self._finished = False
52
+
53
+ def guard(self, step: Callable[[], None]) -> None:
54
+ """Our copy of the response must never break the app's: a failure here drops the
55
+ recording and warns."""
56
+ try:
57
+ step()
58
+ except Exception:
59
+ self.discard()
60
+ warn("a response could not be recorded, so this call is not captured")
61
+
62
+ def discard(self) -> None:
63
+ if self._finished:
64
+ return
65
+ self._finished = True
66
+ self._deliver(None)
67
+
68
+ def chunk(self, data: bytes, arrived: float) -> None:
69
+ if self._first_byte_at is None:
70
+ self._first_byte_at = arrived
71
+ self._text.append(self._decoder.decode(data))
72
+ if self._is_stream:
73
+ self._lines.append(
74
+ {
75
+ "t": "chunk",
76
+ "at_ms": self._since_first_byte(arrived),
77
+ "data": b64encode(data).decode("ascii"),
78
+ }
79
+ )
80
+
81
+ def cancel(self) -> None:
82
+ """The OpenAI client closes the body once it has read `[DONE]`: the stream was complete,
83
+ not abandoned."""
84
+ if DONE.search("".join(self._text)):
85
+ self.complete()
86
+ else:
87
+ self.abort()
88
+
89
+ def complete(self) -> None:
90
+ body = self._body_text()
91
+ if not 200 <= self._status < 300:
92
+ self._lines.append(self._body_line("error", body))
93
+ self._finish("error", [])
94
+ return
95
+ if not self._is_stream:
96
+ self._lines.append(self._body_line("response", body))
97
+ events = events_in(body) if self._is_stream else [_safe_json(body)]
98
+ self._finish("complete", events)
99
+
100
+ def abort(self) -> None:
101
+ self._lines.append({"t": "abort", "at_ms": self._since_first_byte()})
102
+ self._finish("aborted", events_in(self._body_text()))
103
+
104
+ def fail(self, error: BaseException) -> None:
105
+ self._lines.append(self._body_line("error", str(error) or "the response ended early"))
106
+ self._finish("error", events_in(self._body_text()))
107
+
108
+ def _body_line(self, kind: str, body: str) -> dict[str, Any]:
109
+ return {
110
+ "t": kind,
111
+ "at_ms": self._since_first_byte(),
112
+ "status_code": self._status,
113
+ "body": body,
114
+ }
115
+
116
+ # Offsets in the document count from the first byte, so replay can pace itself without the
117
+ # provider's wait.
118
+ def _since_first_byte(self, at: float | None = None) -> int:
119
+ if self._first_byte_at is None:
120
+ return 0
121
+ return _ms((time.perf_counter() if at is None else at) - self._first_byte_at)
122
+
123
+ def _body_text(self) -> str:
124
+ self._text.append(self._decoder.decode(b"", final=True))
125
+ return "".join(self._text)
126
+
127
+ def _finish(self, status: str, events: list[Any]) -> None:
128
+ if self._finished:
129
+ return
130
+ # Marked finished only once the document is built, so a failure here still reaches
131
+ # `discard`.
132
+ document = self._document(status, events)
133
+ self._finished = True
134
+ self._deliver(document)
135
+
136
+ def _document(self, status: str, events: list[Any]) -> str:
137
+ call = self._call
138
+ input_tokens, output_tokens = usage_in(events)
139
+ start = {
140
+ "t": "start",
141
+ "v": 1,
142
+ "trace_id": call.trace.id,
143
+ "first_cache_key": call.trace.first_cache_key,
144
+ "step": call.step,
145
+ "cache_key": call.cache_key,
146
+ "provider": call.provider,
147
+ "model": call.model,
148
+ "is_stream": self._is_stream,
149
+ "request": redact(call.request, call.rules),
150
+ **({"group": call.group} if call.group else {}),
151
+ }
152
+ end = {
153
+ "t": "end",
154
+ "status": status,
155
+ "ttfb_ms": (
156
+ None if self._first_byte_at is None else _ms(self._first_byte_at - call.started_at)
157
+ ),
158
+ "duration_ms": _ms(time.perf_counter() - call.started_at),
159
+ "input_tokens": input_tokens,
160
+ "output_tokens": output_tokens,
161
+ }
162
+ lines = [start, *self._lines, end]
163
+ return "\n".join(json.dumps(line, separators=(",", ":")) for line in lines) + "\n"
164
+
165
+
166
+ def _safe_json(text: str) -> Any:
167
+ try:
168
+ return json.loads(text)
169
+ except ValueError:
170
+ return None
171
+
172
+
173
+ def record_stream(
174
+ call: Call,
175
+ inner: Iterable[bytes],
176
+ deliver: Deliver,
177
+ status: int = 200,
178
+ is_stream: bool = True,
179
+ ) -> Iterator[bytes]:
180
+ """The app gets chunks as the provider sends them; the app leaving early shows up as the
181
+ generator being closed, which is an abort."""
182
+ recording = Recording(call, status, is_stream, deliver)
183
+ try:
184
+ for chunk in inner:
185
+ arrived = time.perf_counter()
186
+ try:
187
+ yield chunk
188
+ finally:
189
+ # Recorded once the app has the chunk (F4), at the time it arrived.
190
+ recording.guard(
191
+ lambda chunk=chunk, arrived=arrived: recording.chunk(chunk, arrived)
192
+ )
193
+ except GeneratorExit:
194
+ recording.guard(recording.cancel)
195
+ raise
196
+ except Exception as error:
197
+ recording.guard(lambda error=error: recording.fail(error))
198
+ raise
199
+ recording.guard(recording.complete)
200
+
201
+
202
+ async def record_astream(
203
+ call: Call,
204
+ inner: AsyncIterable[bytes],
205
+ deliver: Deliver,
206
+ status: int = 200,
207
+ is_stream: bool = True,
208
+ ) -> AsyncIterator[bytes]:
209
+ recording = Recording(call, status, is_stream, deliver)
210
+ try:
211
+ async for chunk in inner:
212
+ arrived = time.perf_counter()
213
+ try:
214
+ yield chunk
215
+ finally:
216
+ # Recorded once the app has the chunk (F4), at the time it arrived.
217
+ recording.guard(
218
+ lambda chunk=chunk, arrived=arrived: recording.chunk(chunk, arrived)
219
+ )
220
+ except (GeneratorExit, asyncio.CancelledError):
221
+ recording.guard(recording.cancel)
222
+ raise
223
+ except Exception as error:
224
+ recording.guard(lambda error=error: recording.fail(error))
225
+ raise
226
+ recording.guard(recording.complete)
agentdevlabs/redact.py ADDED
@@ -0,0 +1,71 @@
1
+ """SI3: the upload copy of a request loses credential fields and anything shaped like a secret.
2
+
3
+ The patterns are spec/redaction/README.md's table, in its order. The cache key is hashed before this
4
+ runs, and responses never pass through it.
5
+ """
6
+
7
+ import re
8
+ from functools import reduce
9
+ from typing import Any
10
+
11
+ REDACTED = "[redacted]"
12
+
13
+ CREDENTIAL_PATHS = [
14
+ path.split(".")
15
+ for path in (
16
+ "tools.*.authorization",
17
+ "tools.*.headers.*",
18
+ "mcp_servers.*.authorization_token",
19
+ )
20
+ ]
21
+
22
+ PATTERNS = [
23
+ re.compile(pattern)
24
+ for pattern in (
25
+ r"-----BEGIN [A-Z ]*PRIVATE KEY( BLOCK)?-----"
26
+ r"([^-]|-[^-])*"
27
+ r"-----END [A-Z ]*PRIVATE KEY( BLOCK)?-----",
28
+ r"adl_(pat|svc)_[A-Za-z0-9]{40}",
29
+ r"sk-(ant|proj|svcacct|admin|None)-[A-Za-z0-9_-]{40,}",
30
+ r"sk-[A-Za-z0-9]{40,}",
31
+ r"[Bb]earer [A-Za-z0-9._~+/=-]{20,}",
32
+ r"(AKIA|ASIA)[A-Z0-9]{16}",
33
+ r"gh[pousr]_[A-Za-z0-9]{36,}",
34
+ r"github_pat_[A-Za-z0-9_]{50,}",
35
+ r"AIza[A-Za-z0-9_-]{35}",
36
+ r"xox[abprs]-[A-Za-z0-9-]{10,}",
37
+ )
38
+ ]
39
+
40
+
41
+ def _redact_path(value: Any, path: list[str]) -> Any:
42
+ if not isinstance(value, (dict, list)):
43
+ return value
44
+ head, rest = path[0], path[1:]
45
+ items = value.items() if isinstance(value, dict) else enumerate(value)
46
+ copied = dict(items) if isinstance(value, dict) else dict(enumerate(value))
47
+ for key in copied:
48
+ if head == "*" or str(key) == head:
49
+ copied[key] = REDACTED if not rest else _redact_path(copied[key], rest)
50
+ return copied if isinstance(value, dict) else list(copied.values())
51
+
52
+
53
+ def _redact_strings(value: Any, patterns: list[re.Pattern]) -> Any:
54
+ if isinstance(value, str):
55
+ return reduce(lambda text, pattern: pattern.sub(REDACTED, text), patterns, value)
56
+ if isinstance(value, list):
57
+ return [_redact_strings(item, patterns) for item in value]
58
+ if isinstance(value, dict):
59
+ return {key: _redact_strings(member, patterns) for key, member in value.items()}
60
+ return value
61
+
62
+
63
+ def redact(request: Any, rules: dict[str, list[str]] | None = None) -> Any:
64
+ """A copy of `request` that is safe to upload.
65
+
66
+ Order: credential fields, the project's fields, the built-in patterns, the project's patterns.
67
+ """
68
+ rules = rules or {}
69
+ paths = CREDENTIAL_PATHS + [path.split(".") for path in rules.get("redact_fields", [])]
70
+ project_patterns = [re.compile(pattern) for pattern in rules.get("redact_patterns", [])]
71
+ return _redact_strings(reduce(_redact_path, paths, request), PATTERNS + project_patterns)