agenthawk 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agenthawk-0.2.0/.github/workflows/ci.yml +22 -0
- agenthawk-0.2.0/.github/workflows/publish-pypi.yml +26 -0
- agenthawk-0.2.0/.gitignore +13 -0
- agenthawk-0.2.0/CONTRIBUTING.md +19 -0
- agenthawk-0.2.0/LICENSE +21 -0
- agenthawk-0.2.0/PKG-INFO +175 -0
- agenthawk-0.2.0/README.md +153 -0
- agenthawk-0.2.0/examples/example_trace.jsonl +15 -0
- agenthawk-0.2.0/pyproject.toml +44 -0
- agenthawk-0.2.0/server.json +23 -0
- agenthawk-0.2.0/src/agenthawk/__init__.py +7 -0
- agenthawk-0.2.0/src/agenthawk/__main__.py +4 -0
- agenthawk-0.2.0/src/agenthawk/core.py +2 -0
- agenthawk-0.2.0/src/agenthawk/server.py +6 -0
- agenthawk-0.2.0/src/mcp_trace/__init__.py +41 -0
- agenthawk-0.2.0/src/mcp_trace/__main__.py +6 -0
- agenthawk-0.2.0/src/mcp_trace/core.py +523 -0
- agenthawk-0.2.0/src/mcp_trace/server.py +163 -0
- agenthawk-0.2.0/tests/conftest.py +6 -0
- agenthawk-0.2.0/tests/fixtures/demo_trace.jsonl +15 -0
- agenthawk-0.2.0/tests/test_core.py +159 -0
- agenthawk-0.2.0/tests/test_server.py +106 -0
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
name: ci
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
|
|
8
|
+
jobs:
|
|
9
|
+
test:
|
|
10
|
+
runs-on: ubuntu-latest
|
|
11
|
+
strategy:
|
|
12
|
+
matrix:
|
|
13
|
+
python-version: ["3.11", "3.12"]
|
|
14
|
+
steps:
|
|
15
|
+
- uses: actions/checkout@v4
|
|
16
|
+
- uses: actions/setup-python@v5
|
|
17
|
+
with:
|
|
18
|
+
python-version: ${{ matrix.python-version }}
|
|
19
|
+
- name: Install
|
|
20
|
+
run: pip install -e ".[dev]"
|
|
21
|
+
- name: Test
|
|
22
|
+
run: pytest -q
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
name: publish-pypi
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
workflow_dispatch:
|
|
5
|
+
release:
|
|
6
|
+
types: [published]
|
|
7
|
+
|
|
8
|
+
permissions:
|
|
9
|
+
contents: read
|
|
10
|
+
id-token: write
|
|
11
|
+
|
|
12
|
+
jobs:
|
|
13
|
+
publish:
|
|
14
|
+
runs-on: ubuntu-latest
|
|
15
|
+
environment: pypi
|
|
16
|
+
steps:
|
|
17
|
+
- uses: actions/checkout@v4
|
|
18
|
+
- uses: actions/setup-python@v5
|
|
19
|
+
with:
|
|
20
|
+
python-version: "3.12"
|
|
21
|
+
- name: Build distributions
|
|
22
|
+
run: |
|
|
23
|
+
python -m pip install --upgrade build
|
|
24
|
+
python -m build
|
|
25
|
+
- name: Publish to PyPI with trusted publishing
|
|
26
|
+
uses: pypa/gh-action-pypi-publish@release/v1
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
# Contributing to AgentHawk
|
|
2
|
+
|
|
3
|
+
## Development setup
|
|
4
|
+
|
|
5
|
+
```bash
|
|
6
|
+
uv venv
|
|
7
|
+
uv pip install -e ".[dev]"
|
|
8
|
+
pytest -q
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
The core query engine is dependency-light and should remain pure. Server tests exercise the MCP 2.x protocol adapter as well as the core functions.
|
|
12
|
+
|
|
13
|
+
## Pull requests
|
|
14
|
+
|
|
15
|
+
- Add a deterministic fixture for new query behavior.
|
|
16
|
+
- Test the tool through `MCPServer.call_tool`, not only by calling the underlying function.
|
|
17
|
+
- Keep tool descriptions phrased as when-to-use prompts.
|
|
18
|
+
- Preserve trace-id prefix support and compact output defaults.
|
|
19
|
+
- Update the README's tool table and limitations when behavior changes.
|
agenthawk-0.2.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Abhishek Ash
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
agenthawk-0.2.0/PKG-INFO
ADDED
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: agenthawk
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Agent observability MCP server for querying OpenTelemetry traces, failures, tool health, security evidence, and regressions.
|
|
5
|
+
Project-URL: Homepage, https://github.com/abhishekash/agenthawk
|
|
6
|
+
Project-URL: Repository, https://github.com/abhishekash/agenthawk
|
|
7
|
+
Project-URL: Issues, https://github.com/abhishekash/agenthawk/issues
|
|
8
|
+
Author-email: Abhishek Ash <ash.abhishek@gmail.com>
|
|
9
|
+
License: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: agent-evals,ai-agents,mcp,model-context-protocol,observability,opentelemetry
|
|
12
|
+
Classifier: Development Status :: 3 - Alpha
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
17
|
+
Requires-Python: >=3.11
|
|
18
|
+
Requires-Dist: mcp>=2.0.0
|
|
19
|
+
Provides-Extra: dev
|
|
20
|
+
Requires-Dist: pytest>=8; extra == 'dev'
|
|
21
|
+
Description-Content-Type: text/markdown
|
|
22
|
+
|
|
23
|
+
# AgentHawk
|
|
24
|
+
|
|
25
|
+
<!-- mcp-name: io.github.abhishekash/agenthawk -->
|
|
26
|
+
|
|
27
|
+
[](https://github.com/abhishekash/agenthawk/actions/workflows/ci.yml) [](LICENSE)
|
|
28
|
+
|
|
29
|
+
**Agents that can debug themselves.** An MCP server that exposes your agent runs — stored as plain OpenTelemetry JSONL span files — as queryable tools: runs, span trees, failures, tool health, security evidence, live activity, regressions, human-approval logs, token/cost usage.
|
|
30
|
+
|
|
31
|
+
The idea: observability shouldn't be a dashboard you read after the fact. It should be *tools your agent can call mid-run* — "why was I slow yesterday?", "what did the human deny me last time?", "which tool keeps timing out?" — or query interactively from Claude Desktop / pi / any MCP client.
|
|
32
|
+
|
|
33
|
+
Pairs with [agent-harness](https://github.com/abhishekash/agent-harness) (which writes the traces), but the reader is format-simple: any JSONL of OTel-shaped spans works.
|
|
34
|
+
|
|
35
|
+
## AI-native use cases
|
|
36
|
+
|
|
37
|
+
| Question during an agent run | MCP tool | Evidence returned |
|
|
38
|
+
|---|---|---|
|
|
39
|
+
| "Why did yesterday's run stall?" | `list_runs` → `slowest_spans` | Run IDs and the longest model or tool spans. |
|
|
40
|
+
| "Did the agent act after I denied the write?" | `approval_log` → `span_tree` | The recorded decision and subsequent execution path. |
|
|
41
|
+
| "How many model tokens did this run use?" | `token_usage` → `span_tree` | Run-level usage totals and the span tree for context. |
|
|
42
|
+
| "Which tool is failing or timing out?" | `failure_report` / `tool_stats` | Redacted diagnostics, error rates, denials, and p95 latency. |
|
|
43
|
+
| "Who/what was allowed to act?" | `security_audit` | Caller, tenant, server provenance, risk, and missing-evidence findings. |
|
|
44
|
+
| "What is happening in the run right now?" | `recent_activity` | Cursor-based polling of newly written spans. |
|
|
45
|
+
| "Did this model/version regress?" | `compare_runs` | Cost, latency, tokens, tool calls, and failure deltas. |
|
|
46
|
+
|
|
47
|
+
The tools read stored traces and can poll files being written, but they do not push events or infer intent from the final answer. See the [real trace fixture](examples/example_trace.jsonl) and the [tool descriptions](src/mcp_trace/server.py) for the exact query contract.
|
|
48
|
+
|
|
49
|
+
## Install & run
|
|
50
|
+
|
|
51
|
+
AgentHawk is the new canonical name. The legacy `abhishekash-mcp-trace` distribution and `mcp-trace` commands remain compatibility aliases for existing users.
|
|
52
|
+
|
|
53
|
+
```bash
|
|
54
|
+
uvx agenthawk --trace-dir ./traces
|
|
55
|
+
# or, for local development:
|
|
56
|
+
git clone https://github.com/abhishekash/agenthawk
|
|
57
|
+
cd agenthawk && uv pip install -e .
|
|
58
|
+
agenthawk --trace-dir ./traces
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
The `agenthawk` 0.2.0 release is the renamed successor to the published
|
|
62
|
+
[`abhishekash-mcp-trace` 0.1.1](https://pypi.org/project/abhishekash-mcp-trace/0.1.1/)
|
|
63
|
+
distribution. Existing clients can continue using the legacy command while
|
|
64
|
+
migrating.
|
|
65
|
+
|
|
66
|
+
## Client configuration
|
|
67
|
+
|
|
68
|
+
**Claude Desktop** (`claude_desktop_config.json`):
|
|
69
|
+
|
|
70
|
+
```json
|
|
71
|
+
{
|
|
72
|
+
"mcpServers": {
|
|
73
|
+
"agent-traces": {
|
|
74
|
+
"command": "uvx",
|
|
75
|
+
"args": ["agenthawk", "--trace-dir", "/path/to/traces"]
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
**pi** (`~/.pi/agent/settings.json`):
|
|
82
|
+
|
|
83
|
+
```json
|
|
84
|
+
{
|
|
85
|
+
"mcpServers": {
|
|
86
|
+
"agent-traces": {
|
|
87
|
+
"command": "uvx",
|
|
88
|
+
"args": ["agenthawk", "--trace-dir", "/path/to/traces"]
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
**agent-harness** (mounted as gated tools):
|
|
95
|
+
|
|
96
|
+
```bash
|
|
97
|
+
harness run "Why was my last run slow?" --mcp "uvx agenthawk --trace-dir ./traces"
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
## Tools
|
|
101
|
+
|
|
102
|
+
| Tool | Use it when |
|
|
103
|
+
|---|---|
|
|
104
|
+
| `list_runs` | Starting out — recent runs with task, model, duration, cost, decision counts |
|
|
105
|
+
| `run_summary` | One run at a glance (accepts trace-id prefix) |
|
|
106
|
+
| `span_tree` | "What did the agent actually do?" — nested shape of the run |
|
|
107
|
+
| `slowest_spans` | "Why was it slow?" — top-k spans by duration |
|
|
108
|
+
| `approval_log` | HITL audit — every approve/deny/**edit**, who decided, and *the rationale* |
|
|
109
|
+
| `token_usage` | Cost questions — aggregated across runs or per-run |
|
|
110
|
+
| `search_spans` | Find spans by tool name, file path, "denied", … |
|
|
111
|
+
| `failure_report` | Find actionable, bounded, redacted errors and failed tool calls. |
|
|
112
|
+
| `tool_stats` | Rank tools by volume, error rate, denials, and latency. |
|
|
113
|
+
| `security_audit` | Audit caller identity, tenant, server provenance, risk, and approval evidence. |
|
|
114
|
+
| `recent_activity` | Poll new spans with a cursor while a run is active. |
|
|
115
|
+
| `compare_runs` | Compare selected traces for regressions across models or versions. |
|
|
116
|
+
|
|
117
|
+
Tool descriptions are written as prompts (when-to-use, not just what-it-does) — descriptions are the interface for agent-called tools.
|
|
118
|
+
|
|
119
|
+
## Example session (real fixture trace)
|
|
120
|
+
|
|
121
|
+
```
|
|
122
|
+
> list_runs
|
|
123
|
+
[{ "trace_id": "f920798dd255…", "task": "Summarize the workspace's notes…",
|
|
124
|
+
"tool_calls": 4, "human_decisions": 2, "stopped_reason": "completed" }]
|
|
125
|
+
|
|
126
|
+
> approval_log
|
|
127
|
+
[{ "tool": "write_file", "decision": "approve", "approver": "auto", … },
|
|
128
|
+
{ "tool": "run_shell", "decision": "approve", "approver": "auto", … }]
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
## Design
|
|
132
|
+
|
|
133
|
+
```
|
|
134
|
+
traces/*.jsonl ──▶ agenthawk.core (pure query functions, zero deps)
|
|
135
|
+
│
|
|
136
|
+
agenthawk.server (thin MCPServer adapter, mcp 2.x)
|
|
137
|
+
│
|
|
138
|
+
stdio (NDJSON JSON-RPC)
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
- **core/server split**: all logic is pure functions over parsed spans; the MCP layer only parses args and JSON-encodes results. Tests hit both layers.
|
|
142
|
+
- **trace_id prefixes**: agents fumble full 32-char hex ids; every tool accepts prefixes.
|
|
143
|
+
- **bounded output**: diagnostics are truncated and obvious credentials are redacted before query results leave the server.
|
|
144
|
+
- **cursor polling**: `recent_activity` makes the snapshot reader useful while a run is still writing spans.
|
|
145
|
+
- The demo fixture ([`examples/example_trace.jsonl`](examples/example_trace.jsonl)) is a *real* agent-harness run, not hand-written.
|
|
146
|
+
|
|
147
|
+
## Research-driven gaps addressed
|
|
148
|
+
|
|
149
|
+
A small Reddit review surfaced the same production problems repeatedly: auth and
|
|
150
|
+
identity are unclear after the demo, versions and logs are hard to compare,
|
|
151
|
+
tool fleets become noisy and expensive, operators lack visibility into what is
|
|
152
|
+
happening, and raw logs are not a useful analysis surface. Examples:
|
|
153
|
+
|
|
154
|
+
- [ChatGPT + MCP gets painful after the demo](https://www.reddit.com/r/ChatGPTPro/comments/1uz6tzs/where_chatgpt_mcp_gets_painful_after_the_demo/) — auth, versions, logs, and safe tools.
|
|
155
|
+
- [MCP logging and correlation IDs](https://www.reddit.com/r/softwarearchitecture/comments/1r2wnfd/is_mcp_effectively_introducing_a_probabilistic/o51k4zq/) — preserve intent, tool arguments, results, and correlation IDs.
|
|
156
|
+
- [180 tools becomes a permission/context/debugging problem](https://www.reddit.com/r/ClaudeAI/comments/1tuqqpn/i_ship_ai_agents_in_production_the_mess_is_mcp/opbh78y/).
|
|
157
|
+
- [MCP security](https://www.reddit.com/r/cybersecurity/comments/1tgs4gg/mcp_security/) — identity, access, credentials, approved versions, and audit logs.
|
|
158
|
+
- [Raw logs are noisy and hard to query](https://www.reddit.com/r/ChatGPTPro/comments/1ur2tx4/i_made_my_codex_usage_tracker_more_agentnative/).
|
|
159
|
+
|
|
160
|
+
This update adds five focused query surfaces rather than pretending a
|
|
161
|
+
Dashboard solves those problems: `failure_report`, `tool_stats`,
|
|
162
|
+
`security_audit`, `recent_activity`, and `compare_runs`. Trace discovery is also
|
|
163
|
+
recursive, and query results redact obvious credential patterns.
|
|
164
|
+
|
|
165
|
+
## Honest limitations
|
|
166
|
+
|
|
167
|
+
- stdio transport only (Streamable HTTP plus authenticated remote access is the next transport boundary)
|
|
168
|
+
- live polling re-reads snapshots; it is not a push subscription
|
|
169
|
+
- security audit can only report identity/provenance that the trace producer records
|
|
170
|
+
- read-only tools; trace mutation (annotations) is roadmap
|
|
171
|
+
- `mcp_trace` remains the compatibility Python import; new code should use `agenthawk`
|
|
172
|
+
|
|
173
|
+
## License
|
|
174
|
+
|
|
175
|
+
MIT
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
# AgentHawk
|
|
2
|
+
|
|
3
|
+
<!-- mcp-name: io.github.abhishekash/agenthawk -->
|
|
4
|
+
|
|
5
|
+
[](https://github.com/abhishekash/agenthawk/actions/workflows/ci.yml) [](LICENSE)
|
|
6
|
+
|
|
7
|
+
**Agents that can debug themselves.** An MCP server that exposes your agent runs — stored as plain OpenTelemetry JSONL span files — as queryable tools: runs, span trees, failures, tool health, security evidence, live activity, regressions, human-approval logs, token/cost usage.
|
|
8
|
+
|
|
9
|
+
The idea: observability shouldn't be a dashboard you read after the fact. It should be *tools your agent can call mid-run* — "why was I slow yesterday?", "what did the human deny me last time?", "which tool keeps timing out?" — or query interactively from Claude Desktop / pi / any MCP client.
|
|
10
|
+
|
|
11
|
+
Pairs with [agent-harness](https://github.com/abhishekash/agent-harness) (which writes the traces), but the reader is format-simple: any JSONL of OTel-shaped spans works.
|
|
12
|
+
|
|
13
|
+
## AI-native use cases
|
|
14
|
+
|
|
15
|
+
| Question during an agent run | MCP tool | Evidence returned |
|
|
16
|
+
|---|---|---|
|
|
17
|
+
| "Why did yesterday's run stall?" | `list_runs` → `slowest_spans` | Run IDs and the longest model or tool spans. |
|
|
18
|
+
| "Did the agent act after I denied the write?" | `approval_log` → `span_tree` | The recorded decision and subsequent execution path. |
|
|
19
|
+
| "How many model tokens did this run use?" | `token_usage` → `span_tree` | Run-level usage totals and the span tree for context. |
|
|
20
|
+
| "Which tool is failing or timing out?" | `failure_report` / `tool_stats` | Redacted diagnostics, error rates, denials, and p95 latency. |
|
|
21
|
+
| "Who/what was allowed to act?" | `security_audit` | Caller, tenant, server provenance, risk, and missing-evidence findings. |
|
|
22
|
+
| "What is happening in the run right now?" | `recent_activity` | Cursor-based polling of newly written spans. |
|
|
23
|
+
| "Did this model/version regress?" | `compare_runs` | Cost, latency, tokens, tool calls, and failure deltas. |
|
|
24
|
+
|
|
25
|
+
The tools read stored traces and can poll files being written, but they do not push events or infer intent from the final answer. See the [real trace fixture](examples/example_trace.jsonl) and the [tool descriptions](src/mcp_trace/server.py) for the exact query contract.
|
|
26
|
+
|
|
27
|
+
## Install & run
|
|
28
|
+
|
|
29
|
+
AgentHawk is the new canonical name. The legacy `abhishekash-mcp-trace` distribution and `mcp-trace` commands remain compatibility aliases for existing users.
|
|
30
|
+
|
|
31
|
+
```bash
|
|
32
|
+
uvx agenthawk --trace-dir ./traces
|
|
33
|
+
# or, for local development:
|
|
34
|
+
git clone https://github.com/abhishekash/agenthawk
|
|
35
|
+
cd agenthawk && uv pip install -e .
|
|
36
|
+
agenthawk --trace-dir ./traces
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
The `agenthawk` 0.2.0 release is the renamed successor to the published
|
|
40
|
+
[`abhishekash-mcp-trace` 0.1.1](https://pypi.org/project/abhishekash-mcp-trace/0.1.1/)
|
|
41
|
+
distribution. Existing clients can continue using the legacy command while
|
|
42
|
+
migrating.
|
|
43
|
+
|
|
44
|
+
## Client configuration
|
|
45
|
+
|
|
46
|
+
**Claude Desktop** (`claude_desktop_config.json`):
|
|
47
|
+
|
|
48
|
+
```json
|
|
49
|
+
{
|
|
50
|
+
"mcpServers": {
|
|
51
|
+
"agent-traces": {
|
|
52
|
+
"command": "uvx",
|
|
53
|
+
"args": ["agenthawk", "--trace-dir", "/path/to/traces"]
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
**pi** (`~/.pi/agent/settings.json`):
|
|
60
|
+
|
|
61
|
+
```json
|
|
62
|
+
{
|
|
63
|
+
"mcpServers": {
|
|
64
|
+
"agent-traces": {
|
|
65
|
+
"command": "uvx",
|
|
66
|
+
"args": ["agenthawk", "--trace-dir", "/path/to/traces"]
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
**agent-harness** (mounted as gated tools):
|
|
73
|
+
|
|
74
|
+
```bash
|
|
75
|
+
harness run "Why was my last run slow?" --mcp "uvx agenthawk --trace-dir ./traces"
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
## Tools
|
|
79
|
+
|
|
80
|
+
| Tool | Use it when |
|
|
81
|
+
|---|---|
|
|
82
|
+
| `list_runs` | Starting out — recent runs with task, model, duration, cost, decision counts |
|
|
83
|
+
| `run_summary` | One run at a glance (accepts trace-id prefix) |
|
|
84
|
+
| `span_tree` | "What did the agent actually do?" — nested shape of the run |
|
|
85
|
+
| `slowest_spans` | "Why was it slow?" — top-k spans by duration |
|
|
86
|
+
| `approval_log` | HITL audit — every approve/deny/**edit**, who decided, and *the rationale* |
|
|
87
|
+
| `token_usage` | Cost questions — aggregated across runs or per-run |
|
|
88
|
+
| `search_spans` | Find spans by tool name, file path, "denied", … |
|
|
89
|
+
| `failure_report` | Find actionable, bounded, redacted errors and failed tool calls. |
|
|
90
|
+
| `tool_stats` | Rank tools by volume, error rate, denials, and latency. |
|
|
91
|
+
| `security_audit` | Audit caller identity, tenant, server provenance, risk, and approval evidence. |
|
|
92
|
+
| `recent_activity` | Poll new spans with a cursor while a run is active. |
|
|
93
|
+
| `compare_runs` | Compare selected traces for regressions across models or versions. |
|
|
94
|
+
|
|
95
|
+
Tool descriptions are written as prompts (when-to-use, not just what-it-does) — descriptions are the interface for agent-called tools.
|
|
96
|
+
|
|
97
|
+
## Example session (real fixture trace)
|
|
98
|
+
|
|
99
|
+
```
|
|
100
|
+
> list_runs
|
|
101
|
+
[{ "trace_id": "f920798dd255…", "task": "Summarize the workspace's notes…",
|
|
102
|
+
"tool_calls": 4, "human_decisions": 2, "stopped_reason": "completed" }]
|
|
103
|
+
|
|
104
|
+
> approval_log
|
|
105
|
+
[{ "tool": "write_file", "decision": "approve", "approver": "auto", … },
|
|
106
|
+
{ "tool": "run_shell", "decision": "approve", "approver": "auto", … }]
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
## Design
|
|
110
|
+
|
|
111
|
+
```
|
|
112
|
+
traces/*.jsonl ──▶ agenthawk.core (pure query functions, zero deps)
|
|
113
|
+
│
|
|
114
|
+
agenthawk.server (thin MCPServer adapter, mcp 2.x)
|
|
115
|
+
│
|
|
116
|
+
stdio (NDJSON JSON-RPC)
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
- **core/server split**: all logic is pure functions over parsed spans; the MCP layer only parses args and JSON-encodes results. Tests hit both layers.
|
|
120
|
+
- **trace_id prefixes**: agents fumble full 32-char hex ids; every tool accepts prefixes.
|
|
121
|
+
- **bounded output**: diagnostics are truncated and obvious credentials are redacted before query results leave the server.
|
|
122
|
+
- **cursor polling**: `recent_activity` makes the snapshot reader useful while a run is still writing spans.
|
|
123
|
+
- The demo fixture ([`examples/example_trace.jsonl`](examples/example_trace.jsonl)) is a *real* agent-harness run, not hand-written.
|
|
124
|
+
|
|
125
|
+
## Research-driven gaps addressed
|
|
126
|
+
|
|
127
|
+
A small Reddit review surfaced the same production problems repeatedly: auth and
|
|
128
|
+
identity are unclear after the demo, versions and logs are hard to compare,
|
|
129
|
+
tool fleets become noisy and expensive, operators lack visibility into what is
|
|
130
|
+
happening, and raw logs are not a useful analysis surface. Examples:
|
|
131
|
+
|
|
132
|
+
- [ChatGPT + MCP gets painful after the demo](https://www.reddit.com/r/ChatGPTPro/comments/1uz6tzs/where_chatgpt_mcp_gets_painful_after_the_demo/) — auth, versions, logs, and safe tools.
|
|
133
|
+
- [MCP logging and correlation IDs](https://www.reddit.com/r/softwarearchitecture/comments/1r2wnfd/is_mcp_effectively_introducing_a_probabilistic/o51k4zq/) — preserve intent, tool arguments, results, and correlation IDs.
|
|
134
|
+
- [180 tools becomes a permission/context/debugging problem](https://www.reddit.com/r/ClaudeAI/comments/1tuqqpn/i_ship_ai_agents_in_production_the_mess_is_mcp/opbh78y/).
|
|
135
|
+
- [MCP security](https://www.reddit.com/r/cybersecurity/comments/1tgs4gg/mcp_security/) — identity, access, credentials, approved versions, and audit logs.
|
|
136
|
+
- [Raw logs are noisy and hard to query](https://www.reddit.com/r/ChatGPTPro/comments/1ur2tx4/i_made_my_codex_usage_tracker_more_agentnative/).
|
|
137
|
+
|
|
138
|
+
This update adds five focused query surfaces rather than pretending a
|
|
139
|
+
Dashboard solves those problems: `failure_report`, `tool_stats`,
|
|
140
|
+
`security_audit`, `recent_activity`, and `compare_runs`. Trace discovery is also
|
|
141
|
+
recursive, and query results redact obvious credential patterns.
|
|
142
|
+
|
|
143
|
+
## Honest limitations
|
|
144
|
+
|
|
145
|
+
- stdio transport only (Streamable HTTP plus authenticated remote access is the next transport boundary)
|
|
146
|
+
- live polling re-reads snapshots; it is not a push subscription
|
|
147
|
+
- security audit can only report identity/provenance that the trace producer records
|
|
148
|
+
- read-only tools; trace mutation (annotations) is roadmap
|
|
149
|
+
- `mcp_trace` remains the compatibility Python import; new code should use `agenthawk`
|
|
150
|
+
|
|
151
|
+
## License
|
|
152
|
+
|
|
153
|
+
MIT
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
{"trace_id": "f920798dd2558fcb84f9977736432a8b", "span_id": "047527cfe5a06b79", "parent_span_id": "b3870190725e6c0a", "name": "llm.complete", "start_unix_nano": 1791446657381364374, "end_unix_nano": 1791446657381395204, "status": "UNSET", "attributes": {"llm.model": "scripted", "llm.usage.input_tokens": 92, "llm.usage.output_tokens": 9, "llm.cost_usd": 0.0}, "events": []}
|
|
2
|
+
{"trace_id": "f920798dd2558fcb84f9977736432a8b", "span_id": "cf9ec887533385eb", "parent_span_id": "b3870190725e6c0a", "name": "tool.call", "start_unix_nano": 1791446657381557076, "end_unix_nano": 1791446657381753320, "status": "UNSET", "attributes": {"tool.name": "list_dir", "tool.risk": "read", "tool.call_id": "call_1d4b88534934", "tool.duration_ms": 0.17095799557864666}, "events": []}
|
|
3
|
+
{"trace_id": "f920798dd2558fcb84f9977736432a8b", "span_id": "b3870190725e6c0a", "parent_span_id": "aec12b5c51ce2580", "name": "agent.step", "start_unix_nano": 1791446657381336939, "end_unix_nano": 1791446657381830109, "status": "UNSET", "attributes": {"step.index": 1}, "events": []}
|
|
4
|
+
{"trace_id": "f920798dd2558fcb84f9977736432a8b", "span_id": "3a1883f304c6810a", "parent_span_id": "060258fcbbc7b4b2", "name": "llm.complete", "start_unix_nano": 1791446657381913540, "end_unix_nano": 1791446657381938066, "status": "UNSET", "attributes": {"llm.model": "scripted", "llm.usage.input_tokens": 94, "llm.usage.output_tokens": 9, "llm.cost_usd": 0.0}, "events": []}
|
|
5
|
+
{"trace_id": "f920798dd2558fcb84f9977736432a8b", "span_id": "ec96e325ccf2812b", "parent_span_id": "060258fcbbc7b4b2", "name": "tool.call", "start_unix_nano": 1791446657382001356, "end_unix_nano": 1791446657382097431, "status": "UNSET", "attributes": {"tool.name": "read_file", "tool.risk": "read", "tool.call_id": "call_e67e16d926b0", "tool.duration_ms": 0.08226605132222176}, "events": []}
|
|
6
|
+
{"trace_id": "f920798dd2558fcb84f9977736432a8b", "span_id": "060258fcbbc7b4b2", "parent_span_id": "aec12b5c51ce2580", "name": "agent.step", "start_unix_nano": 1791446657381892888, "end_unix_nano": 1791446657382146922, "status": "UNSET", "attributes": {"step.index": 2}, "events": []}
|
|
7
|
+
{"trace_id": "f920798dd2558fcb84f9977736432a8b", "span_id": "b07c48a63a82f767", "parent_span_id": "0bd568f46e01fc65", "name": "llm.complete", "start_unix_nano": 1791446657382220834, "end_unix_nano": 1791446657382241746, "status": "UNSET", "attributes": {"llm.model": "scripted", "llm.usage.input_tokens": 125, "llm.usage.output_tokens": 9, "llm.cost_usd": 0.0}, "events": []}
|
|
8
|
+
{"trace_id": "f920798dd2558fcb84f9977736432a8b", "span_id": "d6bc66fcba6ba958", "parent_span_id": "0bd568f46e01fc65", "name": "tool.call", "start_unix_nano": 1791446657382294447, "end_unix_nano": 1791446657382449645, "status": "UNSET", "attributes": {"tool.name": "write_file", "tool.risk": "write", "tool.call_id": "call_8a5a534020b7", "hitl.required": true, "tool.duration_ms": 0.11282693594694138}, "events": [{"name": "hitl.decision", "time_unix_nano": 1791446657382326440, "attributes": {"hitl.decision": "approve", "hitl.approver": "auto", "hitl.rationale": "auto-approved (unattended run)"}}]}
|
|
9
|
+
{"trace_id": "f920798dd2558fcb84f9977736432a8b", "span_id": "0bd568f46e01fc65", "parent_span_id": "aec12b5c51ce2580", "name": "agent.step", "start_unix_nano": 1791446657382193796, "end_unix_nano": 1791446657382508737, "status": "UNSET", "attributes": {"step.index": 3}, "events": []}
|
|
10
|
+
{"trace_id": "f920798dd2558fcb84f9977736432a8b", "span_id": "70d5591a02be91d6", "parent_span_id": "e83a45284010e5c7", "name": "llm.complete", "start_unix_nano": 1791446657382582514, "end_unix_nano": 1791446657382605733, "status": "UNSET", "attributes": {"llm.model": "scripted", "llm.usage.input_tokens": 132, "llm.usage.output_tokens": 9, "llm.cost_usd": 0.0}, "events": []}
|
|
11
|
+
{"trace_id": "f920798dd2558fcb84f9977736432a8b", "span_id": "0f176ff0f2ed3f5f", "parent_span_id": "e83a45284010e5c7", "name": "tool.call", "start_unix_nano": 1791446657382663037, "end_unix_nano": 1791446657384990883, "status": "UNSET", "attributes": {"tool.name": "run_shell", "tool.risk": "execute", "tool.call_id": "call_9a19cc1b4ead", "hitl.required": true, "tool.duration_ms": 2.2949520498514175}, "events": [{"name": "hitl.decision", "time_unix_nano": 1791446657382685591, "attributes": {"hitl.decision": "approve", "hitl.approver": "auto", "hitl.rationale": "auto-approved (unattended run)"}}]}
|
|
12
|
+
{"trace_id": "f920798dd2558fcb84f9977736432a8b", "span_id": "e83a45284010e5c7", "parent_span_id": "aec12b5c51ce2580", "name": "agent.step", "start_unix_nano": 1791446657382563796, "end_unix_nano": 1791446657385069850, "status": "UNSET", "attributes": {"step.index": 4}, "events": []}
|
|
13
|
+
{"trace_id": "f920798dd2558fcb84f9977736432a8b", "span_id": "7e5dff5291ab7ee6", "parent_span_id": "7523732e099104c7", "name": "llm.complete", "start_unix_nano": 1791446657385145680, "end_unix_nano": 1791446657385168372, "status": "UNSET", "attributes": {"llm.model": "scripted", "llm.usage.input_tokens": 196, "llm.usage.output_tokens": 14, "llm.cost_usd": 0.0}, "events": []}
|
|
14
|
+
{"trace_id": "f920798dd2558fcb84f9977736432a8b", "span_id": "7523732e099104c7", "parent_span_id": "aec12b5c51ce2580", "name": "agent.step", "start_unix_nano": 1791446657385127184, "end_unix_nano": 1791446657385204759, "status": "UNSET", "attributes": {"step.index": 5}, "events": []}
|
|
15
|
+
{"trace_id": "f920798dd2558fcb84f9977736432a8b", "span_id": "aec12b5c51ce2580", "parent_span_id": null, "name": "agent.run", "start_unix_nano": 1791446657381269974, "end_unix_nano": 1791446657385251046, "status": "UNSET", "attributes": {"agent.task": "Summarize the workspace's notes into SUMMARY.md, then verify it with ls", "agent.max_steps": 12, "agent.tools": ["list_dir", "read_file", "run_shell", "write_file"], "agent.skills": [], "llm.model": "scripted", "agent.steps": 5, "agent.stopped_reason": "completed", "agent.usage.input_tokens": 639, "agent.usage.output_tokens": 50, "agent.cost_usd": 0.0, "agent.approvals": 2}, "events": []}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "agenthawk"
|
|
7
|
+
version = "0.2.0"
|
|
8
|
+
description = "Agent observability MCP server for querying OpenTelemetry traces, failures, tool health, security evidence, and regressions."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = { text = "MIT" }
|
|
11
|
+
requires-python = ">=3.11"
|
|
12
|
+
authors = [{ name = "Abhishek Ash", email = "ash.abhishek@gmail.com" }]
|
|
13
|
+
keywords = ["mcp", "model-context-protocol", "ai-agents", "observability", "opentelemetry", "agent-evals"]
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Development Status :: 3 - Alpha",
|
|
16
|
+
"Intended Audience :: Developers",
|
|
17
|
+
"License :: OSI Approved :: MIT License",
|
|
18
|
+
"Programming Language :: Python :: 3.11",
|
|
19
|
+
"Programming Language :: Python :: 3.12",
|
|
20
|
+
]
|
|
21
|
+
dependencies = [
|
|
22
|
+
"mcp>=2.0.0",
|
|
23
|
+
]
|
|
24
|
+
|
|
25
|
+
[project.optional-dependencies]
|
|
26
|
+
dev = ["pytest>=8"]
|
|
27
|
+
|
|
28
|
+
[project.scripts]
|
|
29
|
+
agenthawk = "mcp_trace.server:main"
|
|
30
|
+
agent-hawk = "mcp_trace.server:main"
|
|
31
|
+
mcp-trace = "mcp_trace.server:main"
|
|
32
|
+
abhishekash-mcp-trace = "mcp_trace.server:main"
|
|
33
|
+
|
|
34
|
+
[project.urls]
|
|
35
|
+
Homepage = "https://github.com/abhishekash/agenthawk"
|
|
36
|
+
Repository = "https://github.com/abhishekash/agenthawk"
|
|
37
|
+
Issues = "https://github.com/abhishekash/agenthawk/issues"
|
|
38
|
+
|
|
39
|
+
[tool.hatch.build.targets.wheel]
|
|
40
|
+
packages = ["src/mcp_trace", "src/agenthawk"]
|
|
41
|
+
|
|
42
|
+
[tool.pytest.ini_options]
|
|
43
|
+
testpaths = ["tests"]
|
|
44
|
+
addopts = "-q"
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://static.modelcontextprotocol.io/schemas/2025-12-11/server.schema.json",
|
|
3
|
+
"name": "io.github.abhishekash/agenthawk",
|
|
4
|
+
"title": "AgentHawk",
|
|
5
|
+
"description": "Query OpenTelemetry traces from agent runs through MCP.",
|
|
6
|
+
"repository": {
|
|
7
|
+
"url": "https://github.com/abhishekash/agenthawk",
|
|
8
|
+
"source": "github"
|
|
9
|
+
},
|
|
10
|
+
"version": "0.2.0",
|
|
11
|
+
"packages": [
|
|
12
|
+
{
|
|
13
|
+
"registryType": "pypi",
|
|
14
|
+
"registryBaseUrl": "https://pypi.org",
|
|
15
|
+
"identifier": "agenthawk",
|
|
16
|
+
"version": "0.2.0",
|
|
17
|
+
"runtimeHint": "uvx",
|
|
18
|
+
"transport": {
|
|
19
|
+
"type": "stdio"
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
]
|
|
23
|
+
}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
"""AgentHawk public import surface.
|
|
2
|
+
|
|
3
|
+
The implementation remains in ``mcp_trace`` so existing integrations keep
|
|
4
|
+
working. New code may import the query functions from ``agenthawk``.
|
|
5
|
+
"""
|
|
6
|
+
from mcp_trace import * # noqa: F401,F403
|
|
7
|
+
from mcp_trace import __version__
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
"""AgentHawk: MCP server over trace query core.
|
|
2
|
+
|
|
3
|
+
The ``mcp_trace`` import remains as a compatibility module for pre-0.2 users.
|
|
4
|
+
"""
|
|
5
|
+
from mcp_trace.core import (
|
|
6
|
+
approval_log,
|
|
7
|
+
compare_runs,
|
|
8
|
+
failure_report,
|
|
9
|
+
group_traces,
|
|
10
|
+
list_runs,
|
|
11
|
+
load_spans,
|
|
12
|
+
load_trace_dir,
|
|
13
|
+
recent_activity,
|
|
14
|
+
run_summary,
|
|
15
|
+
search_spans,
|
|
16
|
+
security_audit,
|
|
17
|
+
slowest_spans,
|
|
18
|
+
span_tree,
|
|
19
|
+
token_usage,
|
|
20
|
+
tool_stats,
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
__version__ = "0.2.0"
|
|
24
|
+
__all__ = [
|
|
25
|
+
"approval_log",
|
|
26
|
+
"compare_runs",
|
|
27
|
+
"failure_report",
|
|
28
|
+
"group_traces",
|
|
29
|
+
"list_runs",
|
|
30
|
+
"load_spans",
|
|
31
|
+
"load_trace_dir",
|
|
32
|
+
"recent_activity",
|
|
33
|
+
"run_summary",
|
|
34
|
+
"search_spans",
|
|
35
|
+
"security_audit",
|
|
36
|
+
"slowest_spans",
|
|
37
|
+
"span_tree",
|
|
38
|
+
"token_usage",
|
|
39
|
+
"tool_stats",
|
|
40
|
+
"__version__",
|
|
41
|
+
]
|