ams-observability 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ams/__init__.py +38 -0
- ams/claude.py +46 -0
- ams/pricing.py +45 -0
- ams/redact.py +31 -0
- ams/schema.py +168 -0
- ams/storage/__init__.py +30 -0
- ams/storage/local.py +35 -0
- ams/storage/s3.py +78 -0
- ams/tracer.py +399 -0
- ams_observability-0.1.0.dist-info/METADATA +180 -0
- ams_observability-0.1.0.dist-info/RECORD +13 -0
- ams_observability-0.1.0.dist-info/WHEEL +4 -0
- ams_observability-0.1.0.dist-info/licenses/LICENSE +21 -0
ams/__init__.py
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
"""AMS — a super simple monitoring system for Claude agents.
|
|
2
|
+
|
|
3
|
+
Capture a whole Claude Agent SDK session end to end — every tool call, every
|
|
4
|
+
subagent and why it was invoked, the model's reasoning, results, timing and
|
|
5
|
+
cost — as one readable JSON object in blob storage.
|
|
6
|
+
|
|
7
|
+
from ams.claude import traced_query
|
|
8
|
+
|
|
9
|
+
async for message in traced_query(prompt="...", options=options):
|
|
10
|
+
...
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from .schema import (
|
|
14
|
+
SCHEMA_VERSION,
|
|
15
|
+
Agent,
|
|
16
|
+
Event,
|
|
17
|
+
EventType,
|
|
18
|
+
Session,
|
|
19
|
+
Status,
|
|
20
|
+
Totals,
|
|
21
|
+
Usage,
|
|
22
|
+
)
|
|
23
|
+
from .tracer import Tracer
|
|
24
|
+
|
|
25
|
+
__version__ = "0.1.0"
|
|
26
|
+
|
|
27
|
+
__all__ = [
|
|
28
|
+
"Tracer",
|
|
29
|
+
"Session",
|
|
30
|
+
"Event",
|
|
31
|
+
"EventType",
|
|
32
|
+
"Status",
|
|
33
|
+
"Totals",
|
|
34
|
+
"Usage",
|
|
35
|
+
"Agent",
|
|
36
|
+
"SCHEMA_VERSION",
|
|
37
|
+
"__version__",
|
|
38
|
+
]
|
ams/claude.py
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
"""Glue between AMS and `claude_agent_sdk`. This is the whole integration
|
|
2
|
+
surface: swap `query` for `traced_query`, or merge `tracer.hooks()` into your
|
|
3
|
+
options if you drive a ClaudeSDKClient yourself."""
|
|
4
|
+
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
from typing import Any, AsyncIterator, Optional
|
|
8
|
+
|
|
9
|
+
from .tracer import Tracer
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def instrument_options(options: Any, tracer: Tracer) -> Any:
|
|
13
|
+
"""Merge AMS hooks into an existing ClaudeAgentOptions, keeping any of yours."""
|
|
14
|
+
merged = dict(getattr(options, "hooks", None) or {})
|
|
15
|
+
for name, matchers in tracer.hooks().items():
|
|
16
|
+
merged[name] = list(merged.get(name, [])) + list(matchers)
|
|
17
|
+
options.hooks = merged
|
|
18
|
+
return options
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
async def traced_query(
|
|
22
|
+
*,
|
|
23
|
+
prompt: Any,
|
|
24
|
+
options: Any = None,
|
|
25
|
+
tracer: Optional[Tracer] = None,
|
|
26
|
+
**tracer_kwargs: Any,
|
|
27
|
+
) -> AsyncIterator[Any]:
|
|
28
|
+
"""Drop-in replacement for `claude_agent_sdk.query` that records the session.
|
|
29
|
+
|
|
30
|
+
from ams.claude import traced_query
|
|
31
|
+
|
|
32
|
+
async for message in traced_query(prompt="...", options=options):
|
|
33
|
+
print(message)
|
|
34
|
+
|
|
35
|
+
Extra keyword args (storage, agent, environment, tags, metadata, redact)
|
|
36
|
+
are forwarded to `Tracer`. On stream completion the session is written to
|
|
37
|
+
storage automatically.
|
|
38
|
+
"""
|
|
39
|
+
from claude_agent_sdk import ClaudeAgentOptions, query
|
|
40
|
+
|
|
41
|
+
tracer = tracer or Tracer(**tracer_kwargs)
|
|
42
|
+
options = options or ClaudeAgentOptions()
|
|
43
|
+
options = instrument_options(options, tracer)
|
|
44
|
+
|
|
45
|
+
async for message in tracer.watch(query(prompt=prompt, options=options)):
|
|
46
|
+
yield message
|
ams/pricing.py
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
"""Token -> USD cost. The Claude Agent SDK already reports `total_cost_usd` on
|
|
2
|
+
the result message, so AMS uses that for the session total. This table is a
|
|
3
|
+
fallback for costing an individual LLM call from its token usage.
|
|
4
|
+
|
|
5
|
+
Prices are USD per million tokens. Update as needed; matching is by substring so
|
|
6
|
+
dated model ids (e.g. `claude-opus-4-8-20260101`) resolve to the right family.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from typing import Optional
|
|
12
|
+
|
|
13
|
+
from .schema import Usage
|
|
14
|
+
|
|
15
|
+
# (input, output, cache_write, cache_read) per million tokens
|
|
16
|
+
_PRICES: dict[str, tuple[float, float, float, float]] = {
|
|
17
|
+
"claude-opus-4": (15.0, 75.0, 18.75, 1.5),
|
|
18
|
+
"claude-sonnet-4": (3.0, 15.0, 3.75, 0.3),
|
|
19
|
+
"claude-haiku-4": (1.0, 5.0, 1.25, 0.1),
|
|
20
|
+
"claude-3-5-haiku": (0.8, 4.0, 1.0, 0.08),
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _match(model: str) -> Optional[tuple[float, float, float, float]]:
|
|
25
|
+
model = model.lower()
|
|
26
|
+
for key, price in _PRICES.items():
|
|
27
|
+
if key in model:
|
|
28
|
+
return price
|
|
29
|
+
return None
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def cost_usd(model: Optional[str], usage: Optional[Usage]) -> Optional[float]:
|
|
33
|
+
if not model or usage is None:
|
|
34
|
+
return None
|
|
35
|
+
price = _match(model)
|
|
36
|
+
if price is None:
|
|
37
|
+
return None
|
|
38
|
+
p_in, p_out, p_cw, p_cr = price
|
|
39
|
+
total = (
|
|
40
|
+
usage.input_tokens * p_in
|
|
41
|
+
+ usage.output_tokens * p_out
|
|
42
|
+
+ usage.cache_creation_input_tokens * p_cw
|
|
43
|
+
+ usage.cache_read_input_tokens * p_cr
|
|
44
|
+
)
|
|
45
|
+
return round(total / 1_000_000, 6)
|
ams/redact.py
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
"""Optional PII redaction. Off by default — AMS captures full detail unless you
|
|
2
|
+
opt in. Turn it on with `Tracer(redact=True)` or `AMS_REDACT=1` when sessions
|
|
3
|
+
may contain sensitive caller data (phone numbers, emails, cards)."""
|
|
4
|
+
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
import re
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
_PATTERNS = [
|
|
11
|
+
(re.compile(r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b"), "[email]"),
|
|
12
|
+
(re.compile(r"\+?\d[\d\s().-]{7,}\d"), "[phone]"),
|
|
13
|
+
(re.compile(r"\b(?:\d[ -]*?){13,16}\b"), "[card]"),
|
|
14
|
+
(re.compile(r"\b\d{3}-\d{2}-\d{4}\b"), "[ssn]"),
|
|
15
|
+
]
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def redact_text(text: str) -> str:
|
|
19
|
+
for pattern, replacement in _PATTERNS:
|
|
20
|
+
text = pattern.sub(replacement, text)
|
|
21
|
+
return text
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def redact(value: Any) -> Any:
|
|
25
|
+
if isinstance(value, str):
|
|
26
|
+
return redact_text(value)
|
|
27
|
+
if isinstance(value, dict):
|
|
28
|
+
return {k: redact(v) for k, v in value.items()}
|
|
29
|
+
if isinstance(value, list):
|
|
30
|
+
return [redact(v) for v in value]
|
|
31
|
+
return value
|
ams/schema.py
ADDED
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
"""The AMS session schema. This file is the data contract.
|
|
2
|
+
|
|
3
|
+
One session = one JSON object. The shape is deliberately flat and typed so a
|
|
4
|
+
session is easy to read by a human and easy to filter by a machine. Field names
|
|
5
|
+
are aligned with the OpenTelemetry GenAI semantic conventions (`gen_ai.*`) where
|
|
6
|
+
a natural equivalent exists, so the data can later be re-emitted as OTLP without
|
|
7
|
+
renaming. See docs/schema.md.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
from enum import Enum
|
|
13
|
+
from typing import Any, Optional
|
|
14
|
+
|
|
15
|
+
from pydantic import BaseModel, Field
|
|
16
|
+
|
|
17
|
+
SCHEMA_VERSION = "1.0"
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class EventType(str, Enum):
|
|
21
|
+
USER_PROMPT = "user_prompt"
|
|
22
|
+
LLM_MESSAGE = "llm_message"
|
|
23
|
+
TOOL_CALL = "tool_call"
|
|
24
|
+
SUBAGENT = "subagent"
|
|
25
|
+
NOTIFICATION = "notification"
|
|
26
|
+
COMPACTION = "compaction"
|
|
27
|
+
ERROR = "error"
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class Status(str, Enum):
|
|
31
|
+
OK = "ok"
|
|
32
|
+
ERROR = "error"
|
|
33
|
+
RUNNING = "running"
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class Usage(BaseModel):
|
|
37
|
+
input_tokens: int = 0
|
|
38
|
+
output_tokens: int = 0
|
|
39
|
+
cache_read_input_tokens: int = 0
|
|
40
|
+
cache_creation_input_tokens: int = 0
|
|
41
|
+
|
|
42
|
+
def add(self, other: "Usage") -> "Usage":
|
|
43
|
+
return Usage(
|
|
44
|
+
input_tokens=self.input_tokens + other.input_tokens,
|
|
45
|
+
output_tokens=self.output_tokens + other.output_tokens,
|
|
46
|
+
cache_read_input_tokens=self.cache_read_input_tokens
|
|
47
|
+
+ other.cache_read_input_tokens,
|
|
48
|
+
cache_creation_input_tokens=self.cache_creation_input_tokens
|
|
49
|
+
+ other.cache_creation_input_tokens,
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
@classmethod
|
|
53
|
+
def from_sdk(cls, usage: Optional[dict[str, Any]]) -> "Usage":
|
|
54
|
+
usage = usage or {}
|
|
55
|
+
return cls(
|
|
56
|
+
input_tokens=usage.get("input_tokens", 0) or 0,
|
|
57
|
+
output_tokens=usage.get("output_tokens", 0) or 0,
|
|
58
|
+
cache_read_input_tokens=usage.get("cache_read_input_tokens", 0) or 0,
|
|
59
|
+
cache_creation_input_tokens=usage.get("cache_creation_input_tokens", 0) or 0,
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
class LLMDetail(BaseModel):
|
|
64
|
+
model: Optional[str] = None
|
|
65
|
+
stop_reason: Optional[str] = None
|
|
66
|
+
text: Optional[str] = None
|
|
67
|
+
thinking: Optional[str] = None
|
|
68
|
+
message_id: Optional[str] = None
|
|
69
|
+
usage: Optional[Usage] = None
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
class ToolDetail(BaseModel):
|
|
73
|
+
name: str
|
|
74
|
+
tool_use_id: Optional[str] = None
|
|
75
|
+
input: Any = None
|
|
76
|
+
result: Any = None
|
|
77
|
+
is_error: bool = False
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
class SubagentDetail(BaseModel):
|
|
81
|
+
agent_id: str
|
|
82
|
+
agent_type: Optional[str] = None
|
|
83
|
+
transcript_path: Optional[str] = None
|
|
84
|
+
invocation_prompt: Optional[str] = None
|
|
85
|
+
invocation_event_id: Optional[str] = None
|
|
86
|
+
usage: Optional[Usage] = None
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
class ErrorDetail(BaseModel):
|
|
90
|
+
type: Optional[str] = None
|
|
91
|
+
message: Optional[str] = None
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
class Event(BaseModel):
|
|
95
|
+
id: str
|
|
96
|
+
seq: int
|
|
97
|
+
parent_id: Optional[str] = None
|
|
98
|
+
type: EventType
|
|
99
|
+
name: str
|
|
100
|
+
start_time: str
|
|
101
|
+
end_time: Optional[str] = None
|
|
102
|
+
duration_ms: Optional[int] = None
|
|
103
|
+
status: Status = Status.OK
|
|
104
|
+
|
|
105
|
+
prompt: Optional[str] = None
|
|
106
|
+
llm: Optional[LLMDetail] = None
|
|
107
|
+
tool: Optional[ToolDetail] = None
|
|
108
|
+
subagent: Optional[SubagentDetail] = None
|
|
109
|
+
error: Optional[ErrorDetail] = None
|
|
110
|
+
note: Optional[str] = None
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
class Totals(BaseModel):
|
|
114
|
+
usage: Usage = Field(default_factory=Usage)
|
|
115
|
+
cost_usd: Optional[float] = None
|
|
116
|
+
llm_calls: int = 0
|
|
117
|
+
tool_calls: int = 0
|
|
118
|
+
subagents: int = 0
|
|
119
|
+
errors: int = 0
|
|
120
|
+
num_turns: Optional[int] = None
|
|
121
|
+
duration_ms: Optional[int] = None
|
|
122
|
+
duration_api_ms: Optional[int] = None
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
class Agent(BaseModel):
|
|
126
|
+
name: Optional[str] = None
|
|
127
|
+
version: Optional[str] = None
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
class Session(BaseModel):
|
|
131
|
+
schema_version: str = SCHEMA_VERSION
|
|
132
|
+
session_id: str
|
|
133
|
+
trace_id: str
|
|
134
|
+
agent: Agent = Field(default_factory=Agent)
|
|
135
|
+
environment: Optional[str] = None
|
|
136
|
+
tags: list[str] = Field(default_factory=list)
|
|
137
|
+
metadata: dict[str, Any] = Field(default_factory=dict)
|
|
138
|
+
|
|
139
|
+
start_time: str
|
|
140
|
+
end_time: Optional[str] = None
|
|
141
|
+
duration_ms: Optional[int] = None
|
|
142
|
+
status: Status = Status.OK
|
|
143
|
+
|
|
144
|
+
totals: Totals = Field(default_factory=Totals)
|
|
145
|
+
events: list[Event] = Field(default_factory=list)
|
|
146
|
+
|
|
147
|
+
def summary(self) -> dict[str, Any]:
|
|
148
|
+
"""A compact, filterable record for a sessions index — no payloads."""
|
|
149
|
+
return {
|
|
150
|
+
"schema_version": self.schema_version,
|
|
151
|
+
"session_id": self.session_id,
|
|
152
|
+
"trace_id": self.trace_id,
|
|
153
|
+
"agent": self.agent.model_dump(exclude_none=True),
|
|
154
|
+
"environment": self.environment,
|
|
155
|
+
"tags": self.tags,
|
|
156
|
+
"metadata": self.metadata,
|
|
157
|
+
"start_time": self.start_time,
|
|
158
|
+
"end_time": self.end_time,
|
|
159
|
+
"duration_ms": self.duration_ms,
|
|
160
|
+
"status": self.status.value,
|
|
161
|
+
"input_tokens": self.totals.usage.input_tokens,
|
|
162
|
+
"output_tokens": self.totals.usage.output_tokens,
|
|
163
|
+
"cost_usd": self.totals.cost_usd,
|
|
164
|
+
"llm_calls": self.totals.llm_calls,
|
|
165
|
+
"tool_calls": self.totals.tool_calls,
|
|
166
|
+
"subagents": self.totals.subagents,
|
|
167
|
+
"errors": self.totals.errors,
|
|
168
|
+
}
|
ams/storage/__init__.py
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
"""Pluggable storage. A backend just needs to accept a finished Session."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
from typing import Protocol, runtime_checkable
|
|
7
|
+
|
|
8
|
+
from ..schema import Session
|
|
9
|
+
from .local import LocalStorage
|
|
10
|
+
from .s3 import S3Storage
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@runtime_checkable
|
|
14
|
+
class Storage(Protocol):
|
|
15
|
+
def put_session(self, session: Session) -> str: ...
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def from_env() -> Storage:
|
|
19
|
+
"""Pick a backend from the environment.
|
|
20
|
+
|
|
21
|
+
Defaults to S3-compatible blob storage. Set `AMS_STORAGE=local` (or leave
|
|
22
|
+
`AMS_S3_BUCKET` unset) to write JSON to a local directory instead.
|
|
23
|
+
"""
|
|
24
|
+
backend = os.environ.get("AMS_STORAGE", "").lower()
|
|
25
|
+
if backend == "local" or (not backend and not os.environ.get("AMS_S3_BUCKET")):
|
|
26
|
+
return LocalStorage.from_env()
|
|
27
|
+
return S3Storage.from_env()
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
__all__ = ["Storage", "S3Storage", "LocalStorage", "from_env"]
|
ams/storage/local.py
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
"""Local-disk storage. Mirrors the S3 layout under a directory so the same
|
|
2
|
+
frontend code can read either. Useful for development and demos."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import json
|
|
7
|
+
import os
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
from ..schema import Session
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class LocalStorage:
|
|
14
|
+
def __init__(self, root: str = "./ams-data"):
|
|
15
|
+
self.root = Path(root)
|
|
16
|
+
|
|
17
|
+
@classmethod
|
|
18
|
+
def from_env(cls) -> "LocalStorage":
|
|
19
|
+
return cls(root=os.environ.get("AMS_LOCAL_DIR", "./ams-data"))
|
|
20
|
+
|
|
21
|
+
def put_session(self, session: Session) -> str:
|
|
22
|
+
date = session.start_time[:10].replace("-", "/")
|
|
23
|
+
session_path = (
|
|
24
|
+
self.root / "sessions" / date / f"{session.session_id}.json"
|
|
25
|
+
)
|
|
26
|
+
index_path = self.root / "index" / f"{session.session_id}.json"
|
|
27
|
+
session_path.parent.mkdir(parents=True, exist_ok=True)
|
|
28
|
+
index_path.parent.mkdir(parents=True, exist_ok=True)
|
|
29
|
+
session_path.write_text(
|
|
30
|
+
session.model_dump_json(exclude_none=True, indent=2), encoding="utf-8"
|
|
31
|
+
)
|
|
32
|
+
index_path.write_text(
|
|
33
|
+
json.dumps(session.summary(), indent=2), encoding="utf-8"
|
|
34
|
+
)
|
|
35
|
+
return str(session_path)
|
ams/storage/s3.py
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
"""S3-compatible storage. Works against AWS S3, Cloudflare R2, MinIO, etc. —
|
|
2
|
+
anything that speaks the S3 API. Point `AMS_S3_ENDPOINT_URL` at your provider.
|
|
3
|
+
|
|
4
|
+
Layout under the bucket:
|
|
5
|
+
{prefix}/sessions/{YYYY}/{MM}/{DD}/{session_id}.json full session
|
|
6
|
+
{prefix}/index/{session_id}.json compact summary
|
|
7
|
+
|
|
8
|
+
The frontend lists the small `index/` objects to build a searchable session
|
|
9
|
+
list, then fetches a full session on demand.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import json
|
|
15
|
+
import os
|
|
16
|
+
from typing import Optional
|
|
17
|
+
|
|
18
|
+
from ..schema import Session
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class S3Storage:
|
|
22
|
+
def __init__(
|
|
23
|
+
self,
|
|
24
|
+
bucket: str,
|
|
25
|
+
prefix: str = "ams",
|
|
26
|
+
endpoint_url: Optional[str] = None,
|
|
27
|
+
region_name: Optional[str] = None,
|
|
28
|
+
client=None,
|
|
29
|
+
):
|
|
30
|
+
self.bucket = bucket
|
|
31
|
+
self.prefix = prefix.strip("/")
|
|
32
|
+
if client is not None:
|
|
33
|
+
self._client = client
|
|
34
|
+
else:
|
|
35
|
+
import boto3 # imported lazily so `ams` stays importable without it
|
|
36
|
+
|
|
37
|
+
self._client = boto3.client(
|
|
38
|
+
"s3", endpoint_url=endpoint_url, region_name=region_name
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
@classmethod
|
|
42
|
+
def from_env(cls) -> "S3Storage":
|
|
43
|
+
bucket = os.environ.get("AMS_S3_BUCKET")
|
|
44
|
+
if not bucket:
|
|
45
|
+
raise RuntimeError(
|
|
46
|
+
"AMS_S3_BUCKET is not set. Set it (and optionally "
|
|
47
|
+
"AMS_S3_ENDPOINT_URL / AMS_S3_PREFIX / AMS_S3_REGION) or use "
|
|
48
|
+
"LocalStorage."
|
|
49
|
+
)
|
|
50
|
+
return cls(
|
|
51
|
+
bucket=bucket,
|
|
52
|
+
prefix=os.environ.get("AMS_S3_PREFIX", "ams"),
|
|
53
|
+
endpoint_url=os.environ.get("AMS_S3_ENDPOINT_URL"),
|
|
54
|
+
region_name=os.environ.get("AMS_S3_REGION")
|
|
55
|
+
or os.environ.get("AWS_REGION"),
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
def _session_key(self, session: Session) -> str:
|
|
59
|
+
date = session.start_time[:10].replace("-", "/")
|
|
60
|
+
return f"{self.prefix}/sessions/{date}/{session.session_id}.json"
|
|
61
|
+
|
|
62
|
+
def _index_key(self, session: Session) -> str:
|
|
63
|
+
return f"{self.prefix}/index/{session.session_id}.json"
|
|
64
|
+
|
|
65
|
+
def put_session(self, session: Session) -> str:
|
|
66
|
+
body = session.model_dump_json(exclude_none=True, indent=2)
|
|
67
|
+
session_key = self._session_key(session)
|
|
68
|
+
self._put(session_key, body)
|
|
69
|
+
self._put(self._index_key(session), json.dumps(session.summary(), indent=2))
|
|
70
|
+
return f"s3://{self.bucket}/{session_key}"
|
|
71
|
+
|
|
72
|
+
def _put(self, key: str, body: str) -> None:
|
|
73
|
+
self._client.put_object(
|
|
74
|
+
Bucket=self.bucket,
|
|
75
|
+
Key=key,
|
|
76
|
+
Body=body.encode("utf-8"),
|
|
77
|
+
ContentType="application/json",
|
|
78
|
+
)
|
ams/tracer.py
ADDED
|
@@ -0,0 +1,399 @@
|
|
|
1
|
+
"""Tracer records one Claude Agent SDK session into one Session object.
|
|
2
|
+
|
|
3
|
+
It gets its data from two places, because no single source has all of it:
|
|
4
|
+
|
|
5
|
+
* Hooks (PreToolUse / PostToolUse / SubagentStart / ... ) give every tool
|
|
6
|
+
call, every subagent invocation, prompts, and their timing.
|
|
7
|
+
* The message stream (AssistantMessage / ResultMessage) gives the model's
|
|
8
|
+
reasoning (thinking blocks), assistant text, token usage and cost.
|
|
9
|
+
|
|
10
|
+
A Tracer instance is one session. Create a fresh one per run (or per
|
|
11
|
+
ClaudeSDKClient). See claude.py for the one-call `traced_query` wrapper.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import logging
|
|
17
|
+
import time
|
|
18
|
+
import uuid
|
|
19
|
+
from datetime import datetime, timezone
|
|
20
|
+
from typing import Any, AsyncIterator, Optional
|
|
21
|
+
|
|
22
|
+
from . import pricing
|
|
23
|
+
from .redact import redact as _redact
|
|
24
|
+
from .schema import (
|
|
25
|
+
Agent,
|
|
26
|
+
ErrorDetail,
|
|
27
|
+
Event,
|
|
28
|
+
EventType,
|
|
29
|
+
LLMDetail,
|
|
30
|
+
Session,
|
|
31
|
+
Status,
|
|
32
|
+
SubagentDetail,
|
|
33
|
+
ToolDetail,
|
|
34
|
+
Totals,
|
|
35
|
+
Usage,
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
logger = logging.getLogger("ams")
|
|
39
|
+
|
|
40
|
+
_SUBAGENT_TOOLS = {"Task", "Agent"}
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _now_iso() -> str:
|
|
44
|
+
return datetime.now(timezone.utc).isoformat()
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _attr(obj: Any, name: str, default: Any = None) -> Any:
|
|
48
|
+
if isinstance(obj, dict):
|
|
49
|
+
return obj.get(name, default)
|
|
50
|
+
return getattr(obj, name, default)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class Tracer:
|
|
54
|
+
def __init__(
|
|
55
|
+
self,
|
|
56
|
+
storage=None,
|
|
57
|
+
*,
|
|
58
|
+
agent: Optional[Agent] = None,
|
|
59
|
+
environment: Optional[str] = None,
|
|
60
|
+
tags: Optional[list[str]] = None,
|
|
61
|
+
metadata: Optional[dict[str, Any]] = None,
|
|
62
|
+
redact: Optional[bool] = None,
|
|
63
|
+
capture_thinking: bool = True,
|
|
64
|
+
):
|
|
65
|
+
self._storage = storage
|
|
66
|
+
self.agent = agent or Agent()
|
|
67
|
+
self.environment = environment
|
|
68
|
+
self.tags = tags or []
|
|
69
|
+
self.metadata = metadata or {}
|
|
70
|
+
if redact is None:
|
|
71
|
+
import os
|
|
72
|
+
|
|
73
|
+
redact = os.environ.get("AMS_REDACT", "").lower() in ("1", "true", "yes")
|
|
74
|
+
self.redact = redact
|
|
75
|
+
self.capture_thinking = capture_thinking
|
|
76
|
+
|
|
77
|
+
self.trace_id = uuid.uuid4().hex
|
|
78
|
+
self.session_id: Optional[str] = None
|
|
79
|
+
self.start_time = _now_iso()
|
|
80
|
+
self._t0 = time.monotonic()
|
|
81
|
+
|
|
82
|
+
self._events: list[Event] = []
|
|
83
|
+
self._seq = 0
|
|
84
|
+
self._open_tools: dict[str, tuple[Event, float]] = {}
|
|
85
|
+
self._open_subagents: dict[str, Event] = {}
|
|
86
|
+
self._pending_invocations: list[Event] = []
|
|
87
|
+
self._result: Optional[Any] = None
|
|
88
|
+
self._finished: Optional[Session] = None
|
|
89
|
+
|
|
90
|
+
@property
|
|
91
|
+
def storage(self):
|
|
92
|
+
if self._storage is None:
|
|
93
|
+
from .storage import from_env
|
|
94
|
+
|
|
95
|
+
self._storage = from_env()
|
|
96
|
+
return self._storage
|
|
97
|
+
|
|
98
|
+
# ---- helpers -----------------------------------------------------------
|
|
99
|
+
|
|
100
|
+
def _next_seq(self) -> int:
|
|
101
|
+
self._seq += 1
|
|
102
|
+
return self._seq
|
|
103
|
+
|
|
104
|
+
def _clean(self, value: Any) -> Any:
|
|
105
|
+
return _redact(value) if self.redact else value
|
|
106
|
+
|
|
107
|
+
def _see_session_id(self, obj: Any) -> None:
|
|
108
|
+
if self.session_id is None:
|
|
109
|
+
sid = _attr(obj, "session_id")
|
|
110
|
+
if sid:
|
|
111
|
+
self.session_id = sid
|
|
112
|
+
|
|
113
|
+
# ---- hooks -------------------------------------------------------------
|
|
114
|
+
|
|
115
|
+
def hooks(self) -> dict[str, list[Any]]:
|
|
116
|
+
"""Hook config to merge into ClaudeAgentOptions(hooks=...).
|
|
117
|
+
|
|
118
|
+
Captures tool calls, subagents and prompts. Pair with `watch()` (or
|
|
119
|
+
feed messages to `record_message`) to also capture reasoning and cost.
|
|
120
|
+
"""
|
|
121
|
+
from claude_agent_sdk import HookMatcher
|
|
122
|
+
|
|
123
|
+
events = [
|
|
124
|
+
"PreToolUse",
|
|
125
|
+
"PostToolUse",
|
|
126
|
+
"PostToolUseFailure",
|
|
127
|
+
"SubagentStart",
|
|
128
|
+
"SubagentStop",
|
|
129
|
+
"UserPromptSubmit",
|
|
130
|
+
"Notification",
|
|
131
|
+
]
|
|
132
|
+
return {name: [HookMatcher(hooks=[self._hook])] for name in events}
|
|
133
|
+
|
|
134
|
+
async def _hook(self, hook_input: Any, tool_use_id: Optional[str], context: Any):
|
|
135
|
+
try:
|
|
136
|
+
self._see_session_id(hook_input)
|
|
137
|
+
name = _attr(hook_input, "hook_event_name")
|
|
138
|
+
handler = {
|
|
139
|
+
"PreToolUse": self._on_pre_tool,
|
|
140
|
+
"PostToolUse": self._on_post_tool,
|
|
141
|
+
"PostToolUseFailure": self._on_tool_failure,
|
|
142
|
+
"SubagentStart": self._on_subagent_start,
|
|
143
|
+
"SubagentStop": self._on_subagent_stop,
|
|
144
|
+
"UserPromptSubmit": self._on_user_prompt,
|
|
145
|
+
"Notification": self._on_notification,
|
|
146
|
+
}.get(name)
|
|
147
|
+
if handler:
|
|
148
|
+
handler(hook_input)
|
|
149
|
+
except Exception: # never let monitoring break the agent
|
|
150
|
+
logger.exception("ams: hook recording failed")
|
|
151
|
+
return {}
|
|
152
|
+
|
|
153
|
+
def _on_pre_tool(self, hi: Any) -> None:
|
|
154
|
+
tool_name = _attr(hi, "tool_name", "")
|
|
155
|
+
tool_use_id = _attr(hi, "tool_use_id")
|
|
156
|
+
agent_id = _attr(hi, "agent_id")
|
|
157
|
+
parent = self._open_subagents.get(agent_id) if agent_id else None
|
|
158
|
+
event = Event(
|
|
159
|
+
id=uuid.uuid4().hex,
|
|
160
|
+
seq=self._next_seq(),
|
|
161
|
+
parent_id=parent.id if parent else None,
|
|
162
|
+
type=EventType.TOOL_CALL,
|
|
163
|
+
name=f"tool:{tool_name}",
|
|
164
|
+
start_time=_now_iso(),
|
|
165
|
+
status=Status.RUNNING,
|
|
166
|
+
tool=ToolDetail(
|
|
167
|
+
name=tool_name,
|
|
168
|
+
tool_use_id=tool_use_id,
|
|
169
|
+
input=self._clean(_attr(hi, "tool_input")),
|
|
170
|
+
),
|
|
171
|
+
)
|
|
172
|
+
if tool_name in _SUBAGENT_TOOLS:
|
|
173
|
+
ti = _attr(hi, "tool_input") or {}
|
|
174
|
+
event.note = "subagent invocation"
|
|
175
|
+
event.tool.input = self._clean(ti)
|
|
176
|
+
self._pending_invocations.append(event)
|
|
177
|
+
if tool_use_id:
|
|
178
|
+
self._open_tools[tool_use_id] = (event, time.monotonic())
|
|
179
|
+
self._events.append(event)
|
|
180
|
+
|
|
181
|
+
def _close_tool(self, hi: Any) -> Optional[tuple[Event, float]]:
|
|
182
|
+
tool_use_id = _attr(hi, "tool_use_id")
|
|
183
|
+
return self._open_tools.pop(tool_use_id, None) if tool_use_id else None
|
|
184
|
+
|
|
185
|
+
def _on_post_tool(self, hi: Any) -> None:
|
|
186
|
+
found = self._close_tool(hi)
|
|
187
|
+
if not found:
|
|
188
|
+
return
|
|
189
|
+
event, t0 = found
|
|
190
|
+
event.end_time = _now_iso()
|
|
191
|
+
event.duration_ms = int((time.monotonic() - t0) * 1000)
|
|
192
|
+
event.status = Status.OK
|
|
193
|
+
if event.tool:
|
|
194
|
+
event.tool.result = self._clean(_attr(hi, "tool_response"))
|
|
195
|
+
|
|
196
|
+
def _on_tool_failure(self, hi: Any) -> None:
|
|
197
|
+
found = self._close_tool(hi)
|
|
198
|
+
if not found:
|
|
199
|
+
return
|
|
200
|
+
event, t0 = found
|
|
201
|
+
event.end_time = _now_iso()
|
|
202
|
+
event.duration_ms = int((time.monotonic() - t0) * 1000)
|
|
203
|
+
event.status = Status.ERROR
|
|
204
|
+
if event.tool:
|
|
205
|
+
event.tool.is_error = True
|
|
206
|
+
event.error = ErrorDetail(type="tool_error", message=_attr(hi, "error"))
|
|
207
|
+
|
|
208
|
+
def _on_subagent_start(self, hi: Any) -> None:
|
|
209
|
+
agent_id = _attr(hi, "agent_id")
|
|
210
|
+
agent_type = _attr(hi, "agent_type")
|
|
211
|
+
invocation = self._pop_invocation(agent_type)
|
|
212
|
+
detail = SubagentDetail(agent_id=agent_id, agent_type=agent_type)
|
|
213
|
+
if invocation:
|
|
214
|
+
detail.invocation_event_id = invocation.id
|
|
215
|
+
ti = invocation.tool.input if invocation.tool else None
|
|
216
|
+
if isinstance(ti, dict):
|
|
217
|
+
detail.invocation_prompt = ti.get("prompt") or ti.get("description")
|
|
218
|
+
event = Event(
|
|
219
|
+
id=uuid.uuid4().hex,
|
|
220
|
+
seq=self._next_seq(),
|
|
221
|
+
type=EventType.SUBAGENT,
|
|
222
|
+
name=f"subagent:{agent_type or 'agent'}",
|
|
223
|
+
start_time=_now_iso(),
|
|
224
|
+
status=Status.RUNNING,
|
|
225
|
+
subagent=detail,
|
|
226
|
+
)
|
|
227
|
+
if agent_id:
|
|
228
|
+
self._open_subagents[agent_id] = event
|
|
229
|
+
self._events.append(event)
|
|
230
|
+
|
|
231
|
+
def _pop_invocation(self, agent_type: Optional[str]) -> Optional[Event]:
|
|
232
|
+
for i, ev in enumerate(self._pending_invocations):
|
|
233
|
+
ti = ev.tool.input if ev.tool else None
|
|
234
|
+
if isinstance(ti, dict) and ti.get("subagent_type") == agent_type:
|
|
235
|
+
return self._pending_invocations.pop(i)
|
|
236
|
+
return self._pending_invocations.pop(0) if self._pending_invocations else None
|
|
237
|
+
|
|
238
|
+
def _on_subagent_stop(self, hi: Any) -> None:
|
|
239
|
+
agent_id = _attr(hi, "agent_id")
|
|
240
|
+
event = self._open_subagents.pop(agent_id, None) if agent_id else None
|
|
241
|
+
if not event or not event.subagent:
|
|
242
|
+
return
|
|
243
|
+
event.end_time = _now_iso()
|
|
244
|
+
if event.start_time and event.end_time:
|
|
245
|
+
event.status = Status.OK
|
|
246
|
+
event.subagent.transcript_path = _attr(hi, "agent_transcript_path")
|
|
247
|
+
|
|
248
|
+
def _on_user_prompt(self, hi: Any) -> None:
|
|
249
|
+
ts = _now_iso()
|
|
250
|
+
self._events.append(
|
|
251
|
+
Event(
|
|
252
|
+
id=uuid.uuid4().hex,
|
|
253
|
+
seq=self._next_seq(),
|
|
254
|
+
type=EventType.USER_PROMPT,
|
|
255
|
+
name="user_prompt",
|
|
256
|
+
start_time=ts,
|
|
257
|
+
end_time=ts,
|
|
258
|
+
prompt=self._clean(_attr(hi, "prompt")),
|
|
259
|
+
)
|
|
260
|
+
)
|
|
261
|
+
|
|
262
|
+
def _on_notification(self, hi: Any) -> None:
|
|
263
|
+
ts = _now_iso()
|
|
264
|
+
self._events.append(
|
|
265
|
+
Event(
|
|
266
|
+
id=uuid.uuid4().hex,
|
|
267
|
+
seq=self._next_seq(),
|
|
268
|
+
type=EventType.NOTIFICATION,
|
|
269
|
+
name="notification",
|
|
270
|
+
start_time=ts,
|
|
271
|
+
end_time=ts,
|
|
272
|
+
note=_attr(hi, "message"),
|
|
273
|
+
)
|
|
274
|
+
)
|
|
275
|
+
|
|
276
|
+
# ---- message stream ----------------------------------------------------
|
|
277
|
+
|
|
278
|
+
def record_message(self, msg: Any) -> None:
|
|
279
|
+
try:
|
|
280
|
+
self._see_session_id(msg)
|
|
281
|
+
kind = type(msg).__name__
|
|
282
|
+
if kind == "AssistantMessage":
|
|
283
|
+
self._on_assistant(msg)
|
|
284
|
+
elif kind == "ResultMessage":
|
|
285
|
+
self._result = msg
|
|
286
|
+
except Exception:
|
|
287
|
+
logger.exception("ams: message recording failed")
|
|
288
|
+
|
|
289
|
+
def _on_assistant(self, msg: Any) -> None:
|
|
290
|
+
text_parts: list[str] = []
|
|
291
|
+
thinking_parts: list[str] = []
|
|
292
|
+
for block in _attr(msg, "content", []) or []:
|
|
293
|
+
block_kind = type(block).__name__
|
|
294
|
+
if block_kind == "TextBlock":
|
|
295
|
+
text_parts.append(_attr(block, "text", "") or "")
|
|
296
|
+
elif block_kind == "ThinkingBlock" and self.capture_thinking:
|
|
297
|
+
thinking_parts.append(_attr(block, "thinking", "") or "")
|
|
298
|
+
text = "\n".join(p for p in text_parts if p) or None
|
|
299
|
+
thinking = "\n".join(p for p in thinking_parts if p) or None
|
|
300
|
+
if text is None and thinking is None and _attr(msg, "usage") is None:
|
|
301
|
+
return
|
|
302
|
+
usage = Usage.from_sdk(_attr(msg, "usage"))
|
|
303
|
+
ts = _now_iso()
|
|
304
|
+
self._events.append(
|
|
305
|
+
Event(
|
|
306
|
+
id=uuid.uuid4().hex,
|
|
307
|
+
seq=self._next_seq(),
|
|
308
|
+
type=EventType.LLM_MESSAGE,
|
|
309
|
+
name="assistant",
|
|
310
|
+
start_time=ts,
|
|
311
|
+
end_time=ts,
|
|
312
|
+
llm=LLMDetail(
|
|
313
|
+
model=_attr(msg, "model"),
|
|
314
|
+
stop_reason=_attr(msg, "stop_reason"),
|
|
315
|
+
text=self._clean(text) if text else None,
|
|
316
|
+
thinking=self._clean(thinking) if thinking else None,
|
|
317
|
+
message_id=_attr(msg, "message_id"),
|
|
318
|
+
usage=usage,
|
|
319
|
+
),
|
|
320
|
+
)
|
|
321
|
+
)
|
|
322
|
+
|
|
323
|
+
async def watch(self, stream: AsyncIterator[Any]) -> AsyncIterator[Any]:
|
|
324
|
+
"""Pass messages through unchanged while recording them; finalize at end."""
|
|
325
|
+
try:
|
|
326
|
+
async for msg in stream:
|
|
327
|
+
self.record_message(msg)
|
|
328
|
+
yield msg
|
|
329
|
+
finally:
|
|
330
|
+
self.finish()
|
|
331
|
+
|
|
332
|
+
# ---- finalize ----------------------------------------------------------
|
|
333
|
+
|
|
334
|
+
def finish(self, status: Optional[Status] = None) -> Session:
|
|
335
|
+
if self._finished is not None:
|
|
336
|
+
return self._finished
|
|
337
|
+
|
|
338
|
+
events = sorted(self._events, key=lambda e: e.seq)
|
|
339
|
+
totals = self._compute_totals(events)
|
|
340
|
+
wall_ms = int((time.monotonic() - self._t0) * 1000)
|
|
341
|
+
if status is None:
|
|
342
|
+
errored = any(e.status == Status.ERROR for e in events)
|
|
343
|
+
if self._result is not None and _attr(self._result, "is_error"):
|
|
344
|
+
errored = True
|
|
345
|
+
status = Status.ERROR if errored else Status.OK
|
|
346
|
+
|
|
347
|
+
session = Session(
|
|
348
|
+
session_id=self.session_id or self.trace_id,
|
|
349
|
+
trace_id=self.trace_id,
|
|
350
|
+
agent=self.agent,
|
|
351
|
+
environment=self.environment,
|
|
352
|
+
tags=self.tags,
|
|
353
|
+
metadata=self.metadata,
|
|
354
|
+
start_time=self.start_time,
|
|
355
|
+
end_time=_now_iso(),
|
|
356
|
+
duration_ms=totals.duration_ms or wall_ms,
|
|
357
|
+
status=status,
|
|
358
|
+
totals=totals,
|
|
359
|
+
events=events,
|
|
360
|
+
)
|
|
361
|
+
self._finished = session
|
|
362
|
+
try:
|
|
363
|
+
location = self.storage.put_session(session)
|
|
364
|
+
logger.info("ams: wrote session %s -> %s", session.session_id, location)
|
|
365
|
+
except Exception:
|
|
366
|
+
logger.exception("ams: failed to persist session %s", session.session_id)
|
|
367
|
+
return session
|
|
368
|
+
|
|
369
|
+
def _compute_totals(self, events: list[Event]) -> Totals:
|
|
370
|
+
totals = Totals()
|
|
371
|
+
summed = Usage()
|
|
372
|
+
fallback_cost = 0.0
|
|
373
|
+
have_cost = False
|
|
374
|
+
for e in events:
|
|
375
|
+
if e.type == EventType.TOOL_CALL:
|
|
376
|
+
totals.tool_calls += 1
|
|
377
|
+
elif e.type == EventType.SUBAGENT:
|
|
378
|
+
totals.subagents += 1
|
|
379
|
+
elif e.type == EventType.LLM_MESSAGE and e.llm:
|
|
380
|
+
totals.llm_calls += 1
|
|
381
|
+
if e.llm.usage:
|
|
382
|
+
summed = summed.add(e.llm.usage)
|
|
383
|
+
c = pricing.cost_usd(e.llm.model, e.llm.usage)
|
|
384
|
+
if c is not None:
|
|
385
|
+
fallback_cost += c
|
|
386
|
+
have_cost = True
|
|
387
|
+
if e.status == Status.ERROR:
|
|
388
|
+
totals.errors += 1
|
|
389
|
+
|
|
390
|
+
if self._result is not None:
|
|
391
|
+
totals.usage = Usage.from_sdk(_attr(self._result, "usage"))
|
|
392
|
+
totals.cost_usd = _attr(self._result, "total_cost_usd")
|
|
393
|
+
totals.num_turns = _attr(self._result, "num_turns")
|
|
394
|
+
totals.duration_ms = _attr(self._result, "duration_ms")
|
|
395
|
+
totals.duration_api_ms = _attr(self._result, "duration_api_ms")
|
|
396
|
+
else:
|
|
397
|
+
totals.usage = summed
|
|
398
|
+
totals.cost_usd = round(fallback_cost, 6) if have_cost else None
|
|
399
|
+
return totals
|
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: ams-observability
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: A super simple monitoring system for Claude agents — full session traces as JSON in blob storage.
|
|
5
|
+
Project-URL: Homepage, https://github.com/mathu97/ams
|
|
6
|
+
Project-URL: Repository, https://github.com/mathu97/ams
|
|
7
|
+
Author: mathu97
|
|
8
|
+
License: MIT
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Keywords: agents,claude,llm,monitoring,observability,tracing
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Topic :: System :: Monitoring
|
|
16
|
+
Classifier: Typing :: Typed
|
|
17
|
+
Requires-Python: >=3.10
|
|
18
|
+
Requires-Dist: boto3>=1.28
|
|
19
|
+
Requires-Dist: pydantic>=2.0
|
|
20
|
+
Provides-Extra: dev
|
|
21
|
+
Requires-Dist: pytest-asyncio>=0.23; extra == 'dev'
|
|
22
|
+
Requires-Dist: pytest>=8.0; extra == 'dev'
|
|
23
|
+
Description-Content-Type: text/markdown
|
|
24
|
+
|
|
25
|
+
# AMS — Agent Monitoring System
|
|
26
|
+
|
|
27
|
+
A **super simple** monitoring system for [Claude agents](https://docs.claude.com/en/api/agent-sdk/overview). Capture a whole Claude Agent SDK session end to end — every tool call, every subagent and *why* it was invoked, the model's reasoning, results, timing, and cost — as **one readable JSON object** in blob storage.
|
|
28
|
+
|
|
29
|
+
No collector, no database, no agent. One JSON file per session in S3-compatible storage. Built to be trivially easy to read and filter (the things that make Arize and friends painful).
|
|
30
|
+
|
|
31
|
+
```python
|
|
32
|
+
from ams.claude import traced_query
|
|
33
|
+
|
|
34
|
+
async for message in traced_query(prompt="Cancel my membership", options=options):
|
|
35
|
+
...
|
|
36
|
+
# session written to storage automatically when the stream ends
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
That's the whole integration. Swap `query` for `traced_query`.
|
|
40
|
+
|
|
41
|
+
## Why
|
|
42
|
+
|
|
43
|
+
We monitor our Claude agents with Arize today, but it's hard to read, and hard to search/filter for a single session. AMS keeps the data model deliberately flat and typed so a session is obvious to a human and easy to query by a machine. Field names follow the [OpenTelemetry GenAI semantic conventions](https://opentelemetry.io/docs/specs/semconv/gen-ai/) (`gen_ai.*`) where there's a natural equivalent, so the data can later be re-emitted as OTLP without renaming.
|
|
44
|
+
|
|
45
|
+
## What it captures
|
|
46
|
+
|
|
47
|
+
A session is one trace of ordered **events**:
|
|
48
|
+
|
|
49
|
+
| Event | Source | Detail captured |
|
|
50
|
+
|---|---|---|
|
|
51
|
+
| `user_prompt` | `UserPromptSubmit` hook | the prompt |
|
|
52
|
+
| `llm_message` | message stream | model, **thinking / chain-of-thought**, assistant text, token usage |
|
|
53
|
+
| `tool_call` | `PreToolUse` + `PostToolUse` / `PostToolUseFailure` hooks | tool name, input, result, error, **timing** |
|
|
54
|
+
| `subagent` | `SubagentStart` / `SubagentStop` hooks | agent type, **why it was invoked** (the prompt), transcript path; child tool calls nest underneath |
|
|
55
|
+
| `notification` | `Notification` hook | message |
|
|
56
|
+
|
|
57
|
+
Plus session **totals**: token usage (incl. cache read/write), cost (USD), turn count, tool/subagent/error counts, and wall-clock + API duration.
|
|
58
|
+
|
|
59
|
+
The Claude Agent SDK has **no built-in OpenTelemetry** — AMS captures everything through hooks and the message stream, which together are the only place this data lives.
|
|
60
|
+
|
|
61
|
+
## Install
|
|
62
|
+
|
|
63
|
+
```bash
|
|
64
|
+
pip install ams-observability # published name; you import it as `ams`
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
Or from a local checkout: `pip install -e .`. S3-compatible storage (boto3) is included by default.
|
|
68
|
+
|
|
69
|
+
Requires Python 3.10+ and the [`claude-agent-sdk`](https://pypi.org/project/claude-agent-sdk/) in your project.
|
|
70
|
+
|
|
71
|
+
## Configure storage
|
|
72
|
+
|
|
73
|
+
S3-compatible storage is the default. Works against **AWS S3, Cloudflare R2, MinIO** — anything speaking the S3 API.
|
|
74
|
+
|
|
75
|
+
```bash
|
|
76
|
+
export AMS_S3_BUCKET=my-agent-traces
|
|
77
|
+
export AMS_S3_PREFIX=ams # optional, default "ams"
|
|
78
|
+
export AMS_S3_ENDPOINT_URL=https://<account>.r2.cloudflarestorage.com # omit for AWS S3
|
|
79
|
+
export AMS_S3_REGION=auto
|
|
80
|
+
# credentials via the standard AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
Or write to local disk for development:
|
|
84
|
+
|
|
85
|
+
```bash
|
|
86
|
+
export AMS_STORAGE=local
|
|
87
|
+
export AMS_LOCAL_DIR=./ams-data # optional
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
### Layout in the bucket
|
|
91
|
+
|
|
92
|
+
```
|
|
93
|
+
{prefix}/sessions/{YYYY}/{MM}/{DD}/{session_id}.json full session
|
|
94
|
+
{prefix}/index/{session_id}.json compact summary (for listing/filtering)
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
The small `index/` objects let a frontend build a searchable session list without opening every full session.
|
|
98
|
+
|
|
99
|
+
## Usage
|
|
100
|
+
|
|
101
|
+
### One-call (drop-in for `query`)
|
|
102
|
+
|
|
103
|
+
```python
|
|
104
|
+
from ams import Agent
|
|
105
|
+
from ams.claude import traced_query
|
|
106
|
+
|
|
107
|
+
async for message in traced_query(
|
|
108
|
+
prompt="...",
|
|
109
|
+
options=options, # your ClaudeAgentOptions
|
|
110
|
+
agent=Agent(name="support-bot", version="2026.06"),
|
|
111
|
+
environment="prod",
|
|
112
|
+
tags=["voice", "cancellation"],
|
|
113
|
+
metadata={"team_id": "t_42"},
|
|
114
|
+
):
|
|
115
|
+
print(message)
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
### With `ClaudeSDKClient`
|
|
119
|
+
|
|
120
|
+
Merge AMS hooks into your options, feed messages to the tracer, and `finish()` when done:
|
|
121
|
+
|
|
122
|
+
```python
|
|
123
|
+
from ams import Tracer
|
|
124
|
+
from ams.claude import instrument_options
|
|
125
|
+
|
|
126
|
+
tracer = Tracer(environment="prod", tags=["chat"])
|
|
127
|
+
options = instrument_options(my_options, tracer)
|
|
128
|
+
|
|
129
|
+
async with ClaudeSDKClient(options=options) as client:
|
|
130
|
+
await client.query("...")
|
|
131
|
+
async for message in client.receive_response():
|
|
132
|
+
tracer.record_message(message)
|
|
133
|
+
|
|
134
|
+
session = tracer.finish()
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
### Custom storage
|
|
138
|
+
|
|
139
|
+
Pass any object with `put_session(session) -> str`:
|
|
140
|
+
|
|
141
|
+
```python
|
|
142
|
+
tracer = Tracer(storage=MyStorage())
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
## Options
|
|
146
|
+
|
|
147
|
+
| Tracer arg / env | Default | Notes |
|
|
148
|
+
|---|---|---|
|
|
149
|
+
| `storage` / `AMS_STORAGE` | S3 | `local` to write to disk |
|
|
150
|
+
| `agent` | — | `Agent(name=..., version=...)` |
|
|
151
|
+
| `environment` | — | e.g. `prod`, `staging` |
|
|
152
|
+
| `tags`, `metadata` | — | free-form, promoted into the index for filtering |
|
|
153
|
+
| `capture_thinking` | `True` | record the model's reasoning blocks |
|
|
154
|
+
| `redact` / `AMS_REDACT` | `False` | opt-in PII redaction (email / phone / card / SSN) |
|
|
155
|
+
|
|
156
|
+
AMS never throws into your agent: hook and storage failures are logged, not raised.
|
|
157
|
+
|
|
158
|
+
## How it works
|
|
159
|
+
|
|
160
|
+
See [`docs/architecture.md`](docs/architecture.md) for the module map and the two-channel design (hooks + message stream) that AMS fuses into one session.
|
|
161
|
+
|
|
162
|
+
## Schema
|
|
163
|
+
|
|
164
|
+
See [`docs/schema.md`](docs/schema.md) for the full session JSON schema with an example. The contract lives in one file: [`ams/schema.py`](ams/schema.py).
|
|
165
|
+
|
|
166
|
+
## Frontend
|
|
167
|
+
|
|
168
|
+
A simple frontend to browse and filter sessions is planned (not built yet). See [`docs/frontend-notes.md`](docs/frontend-notes.md) for the intended design — it reads the `index/` summaries to list sessions and fetches a full session JSON on click.
|
|
169
|
+
|
|
170
|
+
## Development
|
|
171
|
+
|
|
172
|
+
```bash
|
|
173
|
+
python -m venv .venv && . .venv/bin/activate
|
|
174
|
+
pip install -e ".[dev]"
|
|
175
|
+
pytest
|
|
176
|
+
```
|
|
177
|
+
|
|
178
|
+
## License
|
|
179
|
+
|
|
180
|
+
MIT
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
ams/__init__.py,sha256=GfPzrumwb6d5TixSVQoKCX8yCAC8trYOEdJhSiN8r_M,750
|
|
2
|
+
ams/claude.py,sha256=ixfLxNTVI-1ktMc_AMfnxNYirXTu1z2wXIkOUjCZn5w,1572
|
|
3
|
+
ams/pricing.py,sha256=6LDQLE4dqzrHqyTZ-udJa8hnToW0o16dsPtgn74PI6I,1460
|
|
4
|
+
ams/redact.py,sha256=gcCJ0C5Oy293Lz-Tv_vRmPeHSFbqasiZ2gXmWuuAa9I,983
|
|
5
|
+
ams/schema.py,sha256=XjeM4tMaCgEmpcJwZUIwdnF5df7SF8lc8SWWWtnNuK0,5065
|
|
6
|
+
ams/tracer.py,sha256=G7_E17YHTHSTo8zRqL0vcVlSzDBY1oT5sVefm9Xhs8c,14433
|
|
7
|
+
ams/storage/__init__.py,sha256=aI2jSi4VIuBb0Xzv17VrFLjzJdSZ98-z3FBLO-UF0tw,867
|
|
8
|
+
ams/storage/local.py,sha256=ju73073qIQ0JfoUMz0AiYLN9uuvMiDrkv4cYl-lffu8,1174
|
|
9
|
+
ams/storage/s3.py,sha256=vESblWED3QV03hYZsIaBaiSFXLZh1EyywceoTd2tDuA,2656
|
|
10
|
+
ams_observability-0.1.0.dist-info/METADATA,sha256=4vVqgwpR3xVk8vj15TMmpoN65-Q2f792sYkA9qBQsxw,6635
|
|
11
|
+
ams_observability-0.1.0.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
|
|
12
|
+
ams_observability-0.1.0.dist-info/licenses/LICENSE,sha256=9e1x1uOFAuYHnoFNI80KuVFdQkwqgM4UiJQzoIV6FdQ,1064
|
|
13
|
+
ams_observability-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 mathu97
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|