agentloop-profiler 0.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of agentloop-profiler might be problematic. Click here for more details.
- agentloop/__init__.py +68 -0
- agentloop/audit.py +34 -0
- agentloop/autoinstrument.py +102 -0
- agentloop/ci.py +135 -0
- agentloop/cli.py +876 -0
- agentloop/client.py +170 -0
- agentloop/config.py +50 -0
- agentloop/costs.py +15 -0
- agentloop/decorators.py +160 -0
- agentloop/demo.py +283 -0
- agentloop/doctor.py +344 -0
- agentloop/events.py +49 -0
- agentloop/exporters.py +60 -0
- agentloop/findings.py +215 -0
- agentloop/graph.py +312 -0
- agentloop/integrations/__init__.py +1 -0
- agentloop/integrations/crewai.py +144 -0
- agentloop/integrations/langgraph.py +140 -0
- agentloop/integrations/openai.py +369 -0
- agentloop/integrations/openai_agents.py +212 -0
- agentloop/integrations/sdk.py +82 -0
- agentloop/integrations/vercel_ai.py +83 -0
- agentloop/issues.py +88 -0
- agentloop/markdown.py +102 -0
- agentloop/metrics.py +123 -0
- agentloop/optimizer.py +323 -0
- agentloop/otel.py +302 -0
- agentloop/otel_ids.py +81 -0
- agentloop/patches.py +583 -0
- agentloop/plan_export.py +72 -0
- agentloop/quality.py +222 -0
- agentloop/replay.py +424 -0
- agentloop/runtime.py +244 -0
- agentloop/savings.py +135 -0
- agentloop/server.py +292 -0
- agentloop/store.py +840 -0
- agentloop/timing.py +124 -0
- agentloop/tracer.py +356 -0
- agentloop/value.py +243 -0
- agentloop/version.py +3 -0
- agentloop_profiler-0.4.0.dist-info/METADATA +598 -0
- agentloop_profiler-0.4.0.dist-info/RECORD +50 -0
- agentloop_profiler-0.4.0.dist-info/WHEEL +5 -0
- agentloop_profiler-0.4.0.dist-info/entry_points.txt +2 -0
- agentloop_profiler-0.4.0.dist-info/licenses/LICENSE +201 -0
- agentloop_profiler-0.4.0.dist-info/licenses/THIRD_PARTY_LICENSES.md +54 -0
- agentloop_profiler-0.4.0.dist-info/top_level.txt +2 -0
- dashboard/__init__.py +1 -0
- dashboard/app.py +698 -0
- dashboard/value_view.py +101 -0
|
@@ -0,0 +1,598 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: agentloop-profiler
|
|
3
|
+
Version: 0.4.0
|
|
4
|
+
Summary: Profiler and optimization layer for AI agent execution graphs
|
|
5
|
+
Author: Dipesh Babu
|
|
6
|
+
License-Expression: Apache-2.0
|
|
7
|
+
Project-URL: Homepage, https://github.com/dipeshbabu/agentloop
|
|
8
|
+
Project-URL: Documentation, https://github.com/dipeshbabu/agentloop#readme
|
|
9
|
+
Project-URL: Repository, https://github.com/dipeshbabu/agentloop.git
|
|
10
|
+
Project-URL: Issues, https://github.com/dipeshbabu/agentloop/issues
|
|
11
|
+
Project-URL: Changelog, https://github.com/dipeshbabu/agentloop/blob/main/CHANGELOG.md
|
|
12
|
+
Keywords: ai-agents,observability,optimization,profiling,tracing
|
|
13
|
+
Classifier: Development Status :: 4 - Beta
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: Operating System :: OS Independent
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
22
|
+
Classifier: Topic :: Software Development :: Debuggers
|
|
23
|
+
Classifier: Topic :: Software Development :: Testing
|
|
24
|
+
Requires-Python: >=3.10
|
|
25
|
+
Description-Content-Type: text/markdown
|
|
26
|
+
License-File: LICENSE
|
|
27
|
+
License-File: THIRD_PARTY_LICENSES.md
|
|
28
|
+
Requires-Dist: typer>=0.12.0
|
|
29
|
+
Requires-Dist: rich>=13.0.0
|
|
30
|
+
Provides-Extra: dashboard
|
|
31
|
+
Requires-Dist: streamlit>=1.34.0; extra == "dashboard"
|
|
32
|
+
Requires-Dist: pandas>=2.0.0; extra == "dashboard"
|
|
33
|
+
Provides-Extra: server
|
|
34
|
+
Requires-Dist: fastapi>=0.110.0; extra == "server"
|
|
35
|
+
Requires-Dist: uvicorn>=0.27.0; extra == "server"
|
|
36
|
+
Provides-Extra: postgres
|
|
37
|
+
Requires-Dist: psycopg[binary]>=3.3.4; extra == "postgres"
|
|
38
|
+
Provides-Extra: instrumentation
|
|
39
|
+
Requires-Dist: openai>=1.0.0; extra == "instrumentation"
|
|
40
|
+
Provides-Extra: all
|
|
41
|
+
Requires-Dist: streamlit>=1.34.0; extra == "all"
|
|
42
|
+
Requires-Dist: pandas>=2.0.0; extra == "all"
|
|
43
|
+
Requires-Dist: fastapi>=0.110.0; extra == "all"
|
|
44
|
+
Requires-Dist: uvicorn>=0.27.0; extra == "all"
|
|
45
|
+
Requires-Dist: psycopg[binary]>=3.3.4; extra == "all"
|
|
46
|
+
Requires-Dist: openai>=1.0.0; extra == "all"
|
|
47
|
+
Dynamic: license-file
|
|
48
|
+
|
|
49
|
+
# AgentLoop
|
|
50
|
+
|
|
51
|
+
[](https://github.com/dipeshbabu/agentloop/actions/workflows/ci.yml)
|
|
52
|
+
[](https://pypi.org/project/agentloop-profiler/)
|
|
53
|
+
[](https://www.python.org/downloads/)
|
|
54
|
+
[](LICENSE)
|
|
55
|
+
|
|
56
|
+
AgentLoop is a profiler and optimization layer for AI agent execution graphs. It traces model calls, tool calls, retries, token usage, repeated context, and latency bottlenecks, then turns the trace into concrete workflow rewrite recommendations, patch plans, and replay proof.
|
|
57
|
+
|
|
58
|
+
AgentLoop is not memory for agents. It is performance engineering for agent loops.
|
|
59
|
+
|
|
60
|
+
The design premise is simple: agents do not execute like single prompt-response
|
|
61
|
+
apps. They loop, call tools, retry, branch, summarize, and carry context.
|
|
62
|
+
AgentLoop measures that loop, proposes targeted rewrites, and proves whether the
|
|
63
|
+
rewrite improved the result.
|
|
64
|
+
|
|
65
|
+
Trace in. Rewrite plan out. Replay proof in the PR.
|
|
66
|
+
|
|
67
|
+
AgentLoop is under active pre-1.0 development. Public interfaces may evolve,
|
|
68
|
+
with user-facing changes recorded in the [changelog](CHANGELOG.md).
|
|
69
|
+
|
|
70
|
+
## Install
|
|
71
|
+
|
|
72
|
+
The PyPI distribution is named `agentloop-profiler`. The Python import package
|
|
73
|
+
and command-line program remain `agentloop`:
|
|
74
|
+
|
|
75
|
+
```bash
|
|
76
|
+
python -m pip install agentloop-profiler
|
|
77
|
+
agentloop --help
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
```python
|
|
81
|
+
import agentloop
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
For a source checkout, use [`uv`](https://docs.astral.sh/uv/):
|
|
85
|
+
|
|
86
|
+
```bash
|
|
87
|
+
git clone https://github.com/dipeshbabu/agentloop.git
|
|
88
|
+
cd agentloop
|
|
89
|
+
uv sync --locked --all-extras --no-dev
|
|
90
|
+
uv run agentloop --help
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
Shell examples below assume a source checkout and use `uv run` so they execute
|
|
94
|
+
inside the locked project environment. The prefix is optional only when that
|
|
95
|
+
environment is already active.
|
|
96
|
+
|
|
97
|
+
For a development installation, local quality checks, and pull request guidance,
|
|
98
|
+
see [CONTRIBUTING.md](CONTRIBUTING.md).
|
|
99
|
+
|
|
100
|
+
## Quickstart
|
|
101
|
+
|
|
102
|
+
```python
|
|
103
|
+
from agentloop import trace_agent, trace_model_call, trace_tool_call
|
|
104
|
+
|
|
105
|
+
with trace_agent("research_agent") as trace:
|
|
106
|
+
with trace_model_call("plan", model="gpt-4.1", input_tokens=1200, output_tokens=200):
|
|
107
|
+
pass
|
|
108
|
+
with trace_tool_call("search_web"):
|
|
109
|
+
pass
|
|
110
|
+
|
|
111
|
+
trace.print_report()
|
|
112
|
+
trace.export_json("runs/research_agent.json")
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
### Timing semantics
|
|
116
|
+
|
|
117
|
+
`total_runtime_ms` is the end-to-end elapsed duration of a trace, including work
|
|
118
|
+
that was not wrapped in an event. `cumulative_span_time_ms` is the sum of all
|
|
119
|
+
instrumented event durations, so it can exceed elapsed runtime when spans overlap
|
|
120
|
+
or are nested. Native trace JSON stores `ended_at` and the monotonic-clock
|
|
121
|
+
`elapsed_ms`. Timestamped imports use the earliest span start and latest span end.
|
|
122
|
+
Legacy traces without trustworthy end-time metadata fall back to their cumulative
|
|
123
|
+
event duration for compatibility.
|
|
124
|
+
|
|
125
|
+
### Markdown export safety
|
|
126
|
+
|
|
127
|
+
AgentLoop treats trace-derived values as untrusted in every Markdown exporter.
|
|
128
|
+
Names, findings, recommendations, statuses, paths, and other captured strings are
|
|
129
|
+
rendered as escaped single-line text in headings and prose, and table values cannot
|
|
130
|
+
add rows or columns. Inline and fenced code use delimiters longer than any backtick
|
|
131
|
+
run in their content. Raw HTML is escaped in prose and tables and remains literal
|
|
132
|
+
inside code spans and fences. Exporters do not turn trace values into Markdown links.
|
|
133
|
+
|
|
134
|
+
Keep the Markdown renderer's normal safe mode or HTML sanitizer enabled when reports
|
|
135
|
+
are converted to HTML. These guarantees protect AgentLoop's generated structure;
|
|
136
|
+
they do not make separately appended Markdown or renderer extensions trusted.
|
|
137
|
+
|
|
138
|
+
## Framework integrations
|
|
139
|
+
|
|
140
|
+
AgentLoop now includes dependency-free wrappers for the agent frameworks and SDKs teams are already using:
|
|
141
|
+
|
|
142
|
+
- OpenAI SDK: `instrument_openai_client(...)` records model latency and usage from `responses.create` and `chat.completions.create`.
|
|
143
|
+
- LangGraph: `instrument_state_graph(...)` and `trace_runnable(...)` trace graph nodes and compiled runnables.
|
|
144
|
+
- CrewAI: `instrument_crew(...)`, `instrument_agent(...)`, and `instrument_task(...)` trace crew, agent, and task execution methods.
|
|
145
|
+
- Vercel AI SDK / JS agents: export telemetry dictionaries and convert them with `trace_from_vercel_ai_events(...)`.
|
|
146
|
+
|
|
147
|
+
See `docs/INTEGRATIONS.md` for copy-paste examples.
|
|
148
|
+
|
|
149
|
+
OpenAI example:
|
|
150
|
+
|
|
151
|
+
```python
|
|
152
|
+
from openai import OpenAI
|
|
153
|
+
from agentloop import trace_agent
|
|
154
|
+
from agentloop.integrations.openai import instrument_openai_client
|
|
155
|
+
|
|
156
|
+
client = instrument_openai_client(OpenAI())
|
|
157
|
+
|
|
158
|
+
with trace_agent("research_agent") as trace:
|
|
159
|
+
client.responses.create(model="gpt-4.1-mini", input="Research three competitors.")
|
|
160
|
+
|
|
161
|
+
trace.export_json("runs/openai_agent.json")
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
## Run the end-to-end demo
|
|
165
|
+
|
|
166
|
+
```bash
|
|
167
|
+
uv run agentloop demo-all
|
|
168
|
+
uv run agentloop compare
|
|
169
|
+
uv run agentloop demo --kind proof
|
|
170
|
+
uv run agentloop diagnose --path runs/agentloop_proof_baseline.json --out runs/diagnosis.md --json-out runs/diagnosis.json
|
|
171
|
+
uv run agentloop patch --path runs/agentloop_proof_baseline.json --repo . --out runs/patch_plan.md --json-out runs/patch_plan.json
|
|
172
|
+
uv run agentloop replay --baseline runs/research_agent_baseline.json --candidate runs/research_agent_optimized.json --out runs/replay_report.md --json-out runs/replay_report.json
|
|
173
|
+
uv run agentloop quality-report examples/quality_fixtures.json --out runs/quality_report.md --json-out runs/quality_report.json --min-score 0.9
|
|
174
|
+
uv run agentloop audit --out runs/audit.md
|
|
175
|
+
uv run agentloop optimize --out runs/optimization_plan.md --json-out runs/optimization_plan.json
|
|
176
|
+
uv run agentloop ci --out runs/agentloop_ci.md --json-out runs/agentloop_ci.json
|
|
177
|
+
uv run agentloop value-report --out runs/value_report.json --runs-per-month 5000
|
|
178
|
+
uv run agentloop init-store
|
|
179
|
+
uv run agentloop store-trace --path runs/research_agent_baseline.json --project-id demo
|
|
180
|
+
uv run streamlit run dashboard/app.py
|
|
181
|
+
```
|
|
182
|
+
|
|
183
|
+
The dashboard is a local-first control panel backed by the same persistent store
|
|
184
|
+
used by the API server. It includes:
|
|
185
|
+
|
|
186
|
+
- project overview and usage metering
|
|
187
|
+
- stored traces
|
|
188
|
+
- trace event timelines
|
|
189
|
+
- graph-aware optimization plans
|
|
190
|
+
- machine-actionable diagnosis findings
|
|
191
|
+
- persistent optimization queue across stored traces
|
|
192
|
+
- trace-to-patch dry-run plans
|
|
193
|
+
- before/after replay proof and PR comment previews
|
|
194
|
+
- operational value and pricing-scenario reports
|
|
195
|
+
- API key creation
|
|
196
|
+
- trace ingest and upload flow
|
|
197
|
+
- local, HTTP API, and Postgres setup commands
|
|
198
|
+
|
|
199
|
+
You can still run individual demos:
|
|
200
|
+
|
|
201
|
+
```bash
|
|
202
|
+
uv run agentloop demo --kind baseline
|
|
203
|
+
uv run agentloop demo --kind optimized
|
|
204
|
+
uv run agentloop demo --kind langgraph
|
|
205
|
+
uv run agentloop demo --kind proof
|
|
206
|
+
uv run python examples/langgraph_auto_instrumentation_demo.py
|
|
207
|
+
```
|
|
208
|
+
|
|
209
|
+
Only `agentloop demo` and `agentloop demo-all` generate synthetic traces. Their
|
|
210
|
+
output is labeled as synthetic in the terminal and carries
|
|
211
|
+
`metadata.synthetic = true` with `metadata.source = "agentloop_demo"`.
|
|
212
|
+
|
|
213
|
+
Commands that load a trace require the supplied path to be an existing,
|
|
214
|
+
readable regular file containing a valid trace. Missing, unreadable, or malformed
|
|
215
|
+
inputs produce a clear error and a non-zero exit code; they are never replaced
|
|
216
|
+
with demo data. The former `--autogen` fallback has been removed. Run one of the
|
|
217
|
+
demo commands explicitly before using the generated paths in the examples above.
|
|
218
|
+
|
|
219
|
+
## Replay gates
|
|
220
|
+
|
|
221
|
+
```bash
|
|
222
|
+
uv run agentloop replay \
|
|
223
|
+
--baseline runs/research_agent_baseline.json \
|
|
224
|
+
--candidate runs/research_agent_optimized.json \
|
|
225
|
+
--out runs/replay_report.md \
|
|
226
|
+
--json-out runs/replay_report.json \
|
|
227
|
+
--min-latency-improvement-pct 20 \
|
|
228
|
+
--min-cost-improvement-pct 5 \
|
|
229
|
+
--max-latency-regression-pct 0 \
|
|
230
|
+
--max-cost-regression-pct 0 \
|
|
231
|
+
--require-retry-non-increase \
|
|
232
|
+
--quality-fixtures examples/quality_fixtures.json \
|
|
233
|
+
--min-quality-score 0.9
|
|
234
|
+
```
|
|
235
|
+
|
|
236
|
+
`agentloop replay` compares a baseline trace with a candidate trace and turns the
|
|
237
|
+
result into pass/fail gates for CI and PR comments. It reports latency, cost,
|
|
238
|
+
token, retry, tool-call, model-call, schema, and quality deltas, then exits
|
|
239
|
+
non-zero when gates fail unless `--no-fail-on-gate` is used.
|
|
240
|
+
|
|
241
|
+
## Quality gates
|
|
242
|
+
|
|
243
|
+
```json
|
|
244
|
+
{
|
|
245
|
+
"fixtures": [
|
|
246
|
+
{
|
|
247
|
+
"id": "required_summary_fields",
|
|
248
|
+
"input": "Summarize one source.",
|
|
249
|
+
"baseline_output": {"summary": "AgentLoop proves rewrites.", "sources": ["source-a"]},
|
|
250
|
+
"candidate_output": {"summary": "AgentLoop proves rewrites.", "sources": ["source-a"]},
|
|
251
|
+
"scorer": {"type": "required_fields", "required": ["summary", "sources"]}
|
|
252
|
+
}
|
|
253
|
+
]
|
|
254
|
+
}
|
|
255
|
+
```
|
|
256
|
+
|
|
257
|
+
Quality fixtures support dependency-free scorers:
|
|
258
|
+
|
|
259
|
+
- `exact_match`
|
|
260
|
+
- `contains`
|
|
261
|
+
- `glob` with a maximum 256-character pattern
|
|
262
|
+
- `required_fields` / `json_schema`
|
|
263
|
+
- `json_subset`
|
|
264
|
+
- `custom` with `module:function`
|
|
265
|
+
|
|
266
|
+
Use fixtures with `agentloop quality-report`, `agentloop replay --quality-fixtures`,
|
|
267
|
+
or `agentloop ci --quality-fixtures`. This makes the PR proof show whether the
|
|
268
|
+
candidate is faster, cheaper, and still correct.
|
|
269
|
+
|
|
270
|
+
Custom scorers import and execute Python code. Use them only with trusted local
|
|
271
|
+
fixture files; the HTTP `/quality-report` endpoint rejects custom scorers.
|
|
272
|
+
Raw regular-expression scorers are also rejected because caller-controlled
|
|
273
|
+
patterns can cause regular-expression denial of service. Use `glob` or
|
|
274
|
+
`contains` for untrusted pattern matching.
|
|
275
|
+
|
|
276
|
+
## Value reports
|
|
277
|
+
|
|
278
|
+
```bash
|
|
279
|
+
uv run agentloop value-report --path runs/research_agent_baseline.json \
|
|
280
|
+
--out runs/value_report.json \
|
|
281
|
+
--runs-per-month 5000 \
|
|
282
|
+
--engineer-hourly-rate-usd 150 \
|
|
283
|
+
--incident-cost-usd 500
|
|
284
|
+
```
|
|
285
|
+
|
|
286
|
+
A value report converts trace data into operational ROI metrics:
|
|
287
|
+
|
|
288
|
+
- direct model-cost savings per month
|
|
289
|
+
- latency saved per run and per month
|
|
290
|
+
- engineering hours saved from removing repeated bottlenecks
|
|
291
|
+
- reliability risk score based on retries, context duplication, and optimization opportunities
|
|
292
|
+
- optional pricing scenarios and modeled value-to-price ratios
|
|
293
|
+
- a concise summary of measured savings and reliability impact
|
|
294
|
+
|
|
295
|
+
The pricing scenario is intentionally conservative. It maps measured
|
|
296
|
+
monthly value into one of `free`, `pro`, `team`, `growth`, or `enterprise`, then
|
|
297
|
+
shows the suggested monthly price and modeled value-to-price ratio. Treat this
|
|
298
|
+
as a configurable planning estimate rather than a billing or procurement source
|
|
299
|
+
of truth.
|
|
300
|
+
|
|
301
|
+
The Streamlit dashboard includes a dedicated `Value & Pricing` page. Select a
|
|
302
|
+
stored trace, adjust assumptions such as monthly run volume and engineering
|
|
303
|
+
rate, then download the value report JSON for capacity planning or optimization
|
|
304
|
+
prioritization. The Optimization page also has an expandable value estimate so
|
|
305
|
+
the same trace can connect a technical bottleneck to its estimated operational
|
|
306
|
+
impact.
|
|
307
|
+
|
|
308
|
+
Remote value reports are available through both API and CLI:
|
|
309
|
+
|
|
310
|
+
```bash
|
|
311
|
+
uv run agentloop remote-value-report RUN_ID --api-url http://127.0.0.1:8000 --api-key al_xxx
|
|
312
|
+
curl --fail --header "X-AgentLoop-Key: al_xxx" \
|
|
313
|
+
"http://127.0.0.1:8000/traces/RUN_ID/value?runs_per_month=5000"
|
|
314
|
+
```
|
|
315
|
+
|
|
316
|
+
## Optimization plans
|
|
317
|
+
|
|
318
|
+
```bash
|
|
319
|
+
uv run agentloop optimize --path runs/research_agent_baseline.json --out runs/optimization_plan.md --json-out runs/optimization_plan.json
|
|
320
|
+
```
|
|
321
|
+
|
|
322
|
+
AgentLoop reconstructs an execution graph, identifies bottlenecks, and emits optimization cards such as:
|
|
323
|
+
|
|
324
|
+
- parallelize independent tool calls
|
|
325
|
+
- cache repeated prompt/context prefixes
|
|
326
|
+
- batch repeated model calls
|
|
327
|
+
- route small steps to cheaper models
|
|
328
|
+
- reduce retry loops with structured outputs
|
|
329
|
+
- split or compress oversized reasoning steps
|
|
330
|
+
|
|
331
|
+
Each card includes a reason, affected nodes, confidence, rewrite hint, and estimated latency or cost savings.
|
|
332
|
+
|
|
333
|
+
Cards that touch the same spans compete for the same work, so their estimates are
|
|
334
|
+
alternatives rather than additive. The plan's projected totals come from the
|
|
335
|
+
compatible (span-disjoint) subset of cards that maximizes latency savings, with
|
|
336
|
+
ties broken by cost savings, capped at the run's actual runtime and cost — so the
|
|
337
|
+
reported latency/cost pair is always achievable by one concrete set of changes.
|
|
338
|
+
The plan's `savings_aggregation` block records the rule, the selected card
|
|
339
|
+
indexes, and the raw versus effective totals.
|
|
340
|
+
|
|
341
|
+
## Diagnosis and OpenTelemetry interop
|
|
342
|
+
|
|
343
|
+
```bash
|
|
344
|
+
uv run agentloop diagnose --path runs/research_agent_baseline.json --out runs/diagnosis.md --json-out runs/diagnosis.json
|
|
345
|
+
uv run agentloop patch --path runs/research_agent_baseline.json --repo . --out runs/patch_plan.md --json-out runs/patch_plan.json
|
|
346
|
+
uv run agentloop list-findings --project-id demo
|
|
347
|
+
uv run agentloop optimization-queue --project-id demo --json-out runs/optimization_queue.json
|
|
348
|
+
uv run agentloop github-issue-drafts --project-id demo --out runs/agentloop_issue_drafts.md
|
|
349
|
+
uv run agentloop export-otel runs/research_agent_baseline.json runs/research_agent_baseline.otel.json
|
|
350
|
+
uv run agentloop import-otel runs/research_agent_baseline.otel.json runs/imported_trace.json --name imported_agent
|
|
351
|
+
uv run agentloop diagnose --path runs/research_agent_baseline.otel.json --otel --out runs/otel_diagnosis.md
|
|
352
|
+
```
|
|
353
|
+
|
|
354
|
+
`agentloop diagnose` turns optimization opportunities into machine-actionable findings:
|
|
355
|
+
stable finding IDs, affected spans, evidence rows, savings formulas, patchability, and
|
|
356
|
+
replay acceptance criteria. `export-otel` and `import-otel` let AgentLoop sit above
|
|
357
|
+
OpenTelemetry GenAI-style spans instead of requiring a proprietary trace source.
|
|
358
|
+
`agentloop patch --dry-run` turns supported findings into framework-aware patch
|
|
359
|
+
plans for parallel tool execution, context caching, structured-output repair,
|
|
360
|
+
batching, model routing, split/compress rewrites, runaway-loop guardrails, and
|
|
361
|
+
tool-oscillation guards without modifying source files.
|
|
362
|
+
When traces are stored through the local or deployed API, AgentLoop persists the
|
|
363
|
+
diagnosis findings and clusters them into an optimization queue ranked by
|
|
364
|
+
severity, frequency, patchability, and estimated savings. Queue savings use the
|
|
365
|
+
same overlap rule as optimization plans: within one run, findings of the same
|
|
366
|
+
type and title that share affected spans count once (the best compatible
|
|
367
|
+
selection), so repeated detections of the same problem do not inflate
|
|
368
|
+
`priority_score`.
|
|
369
|
+
`agentloop github-issue-drafts` turns the top patchable queue items into
|
|
370
|
+
GitHub-ready issue titles, labels, bodies, and acceptance criteria.
|
|
371
|
+
|
|
372
|
+
## CI and PR performance gates
|
|
373
|
+
|
|
374
|
+
```bash
|
|
375
|
+
uv run agentloop ci \
|
|
376
|
+
--baseline runs/research_agent_baseline.json \
|
|
377
|
+
--candidate runs/research_agent_optimized.json \
|
|
378
|
+
--out runs/agentloop_ci.md \
|
|
379
|
+
--json-out runs/agentloop_ci.json \
|
|
380
|
+
--min-latency-improvement-pct 20 \
|
|
381
|
+
--min-cost-improvement-pct 5 \
|
|
382
|
+
--max-latency-regression-pct 0 \
|
|
383
|
+
--max-cost-regression-pct 0 \
|
|
384
|
+
--quality-fixtures examples/quality_fixtures.json \
|
|
385
|
+
--min-quality-score 0.9
|
|
386
|
+
```
|
|
387
|
+
|
|
388
|
+
`agentloop ci` combines replay gates with candidate diagnosis findings. It emits
|
|
389
|
+
Markdown for PR summaries and JSON for automation, then exits non-zero when the
|
|
390
|
+
configured cost, latency, or retry gates fail. The included GitHub Actions
|
|
391
|
+
workflow runs this against the demo baseline and optimized traces and appends
|
|
392
|
+
the report to the workflow step summary and pull request comments.
|
|
393
|
+
|
|
394
|
+
For a real repository, have your agent test job write trace artifacts, then point
|
|
395
|
+
the workflow at those paths:
|
|
396
|
+
|
|
397
|
+
```bash
|
|
398
|
+
uv run agentloop ci --baseline artifacts/agentloop/baseline.json --candidate artifacts/agentloop/candidate.json
|
|
399
|
+
```
|
|
400
|
+
|
|
401
|
+
The workflow also supports manual inputs for baseline path, candidate path, and
|
|
402
|
+
gate thresholds. Use `examples/ci_pr_trace_demo.py` to generate a local trace
|
|
403
|
+
artifact pair and preview the PR report.
|
|
404
|
+
|
|
405
|
+
## Dashboard
|
|
406
|
+
|
|
407
|
+
```bash
|
|
408
|
+
uv run streamlit run dashboard/app.py
|
|
409
|
+
```
|
|
410
|
+
|
|
411
|
+
Dashboard pages:
|
|
412
|
+
|
|
413
|
+
- `Overview`: usage, run count, runtime, cost, token volume, model/tool calls, retry count, recent runs, runtime chart
|
|
414
|
+
- `Traces`: stored trace table, event timeline, trace JSON download
|
|
415
|
+
- `Optimization Queue`: ranked recurring findings across stored traces
|
|
416
|
+
- `Optimization`: optimization cards, bottlenecks, parallelizable groups, value estimate, plan JSON download
|
|
417
|
+
- `Diagnosis`: evidence-backed findings with severity, affected spans, savings, and validation criteria
|
|
418
|
+
- `Patch Plan`: framework-aware dry-run rewrite plans tied to likely files and replay gates
|
|
419
|
+
- `Replay Proof`: before/after metrics, quality/schema gates, and PR comment preview
|
|
420
|
+
- `Quality Gates`: fixture scoring for exact match, bounded glob matching, JSON fields, JSON subset, and custom scorers
|
|
421
|
+
- `Value & Pricing`: operational ROI, reliability risk, pricing scenarios, and value report JSON download
|
|
422
|
+
- `API Keys`: project-scoped API key creation
|
|
423
|
+
- `Ingest`: generate demo traces, upload trace JSON, store traces under a project
|
|
424
|
+
- `Setup`: local, HTTP API, and Postgres deployment commands
|
|
425
|
+
|
|
426
|
+
See the [dashboard guide](docs/DASHBOARD.md) for setup and operating details.
|
|
427
|
+
|
|
428
|
+
## Local API server
|
|
429
|
+
|
|
430
|
+
```bash
|
|
431
|
+
uv run agentloop server --host 127.0.0.1 --port 8000
|
|
432
|
+
```
|
|
433
|
+
|
|
434
|
+
Endpoints:
|
|
435
|
+
|
|
436
|
+
- `GET /health`
|
|
437
|
+
- `GET /readyz`
|
|
438
|
+
- `POST /api-keys`
|
|
439
|
+
- `POST /traces`
|
|
440
|
+
- `GET /traces`
|
|
441
|
+
- `GET /traces/{run_id}/report`
|
|
442
|
+
- `GET /traces/{run_id}/optimize`
|
|
443
|
+
- `GET /traces/{run_id}/diagnose`
|
|
444
|
+
- `GET /findings`
|
|
445
|
+
- `GET /optimization-queue`
|
|
446
|
+
- `GET /optimization-queue/github-issues`
|
|
447
|
+
- `POST /quality-report`
|
|
448
|
+
- `GET /traces/{run_id}/value`
|
|
449
|
+
- `GET /usage`
|
|
450
|
+
|
|
451
|
+
This is the HTTP API path: SDK traces can be sent to an API, stored, metered,
|
|
452
|
+
and turned into optimization plans and value reports.
|
|
453
|
+
|
|
454
|
+
For production deployment, use Postgres, API-key auth, an admin key for API-key
|
|
455
|
+
creation, and the Docker/Compose scaffolding in `docs/PRODUCTION.md`.
|
|
456
|
+
|
|
457
|
+
```bash
|
|
458
|
+
uv run agentloop production-check
|
|
459
|
+
uv run python scripts/smoke_api.py
|
|
460
|
+
```
|
|
461
|
+
|
|
462
|
+
## Persistent store and metering
|
|
463
|
+
|
|
464
|
+
AgentLoop supports a local SQLite store and a Postgres store for shared or
|
|
465
|
+
self-hosted deployments.
|
|
466
|
+
|
|
467
|
+
SQLite is the default:
|
|
468
|
+
|
|
469
|
+
```bash
|
|
470
|
+
uv run agentloop init-store
|
|
471
|
+
uv run agentloop demo-all
|
|
472
|
+
uv run agentloop store-trace --path runs/research_agent_baseline.json --project-id demo
|
|
473
|
+
uv run agentloop list-stored-traces --project-id demo
|
|
474
|
+
uv run agentloop usage-summary --project-id demo
|
|
475
|
+
```
|
|
476
|
+
|
|
477
|
+
Postgres mode:
|
|
478
|
+
|
|
479
|
+
```bash
|
|
480
|
+
uv sync --locked --extra server --extra postgres --no-dev
|
|
481
|
+
export AGENTLOOP_STORE_BACKEND=postgres
|
|
482
|
+
export AGENTLOOP_DATABASE_URL=postgresql://agentloop:agentloop@localhost:5432/agentloop
|
|
483
|
+
uv run agentloop init-store
|
|
484
|
+
uv run agentloop create-api-key --project-id acme --name local-dev
|
|
485
|
+
uv run agentloop server --host 0.0.0.0 --port 8000
|
|
486
|
+
```
|
|
487
|
+
|
|
488
|
+
Then upload traces with the project API key:
|
|
489
|
+
|
|
490
|
+
```bash
|
|
491
|
+
uv run agentloop upload --path runs/research_agent_baseline.json --api-url http://127.0.0.1:8000 --api-key al_xxx
|
|
492
|
+
uv run agentloop remote-usage --api-url http://127.0.0.1:8000 --api-key al_xxx
|
|
493
|
+
uv run agentloop remote-value-report RUN_ID --api-url http://127.0.0.1:8000 --api-key al_xxx
|
|
494
|
+
```
|
|
495
|
+
|
|
496
|
+
### Optional single-secret API key protection
|
|
497
|
+
|
|
498
|
+
For simple local demos, you can still use one static key:
|
|
499
|
+
|
|
500
|
+
```bash
|
|
501
|
+
export AGENTLOOP_REQUIRE_API_KEY=true
|
|
502
|
+
export AGENTLOOP_API_KEY=dev-secret
|
|
503
|
+
uv run agentloop server --host 127.0.0.1 --port 8000
|
|
504
|
+
```
|
|
505
|
+
|
|
506
|
+
Then upload with:
|
|
507
|
+
|
|
508
|
+
```bash
|
|
509
|
+
uv run agentloop upload --path runs/research_agent_baseline.json --api-url http://127.0.0.1:8000 --api-key dev-secret
|
|
510
|
+
```
|
|
511
|
+
|
|
512
|
+
### Upload traces from Python
|
|
513
|
+
|
|
514
|
+
```python
|
|
515
|
+
from agentloop import AgentLoopClient
|
|
516
|
+
|
|
517
|
+
client = AgentLoopClient(base_url="http://127.0.0.1:8000", api_key="al_xxx")
|
|
518
|
+
response = client.upload_trace("runs/research_agent_baseline.json")
|
|
519
|
+
plan = client.get_optimization_plan(response["run_id"])
|
|
520
|
+
value = client.get_value_report(response["run_id"], runs_per_month=5000)
|
|
521
|
+
usage = client.usage_summary()
|
|
522
|
+
```
|
|
523
|
+
|
|
524
|
+
Or use environment variables:
|
|
525
|
+
|
|
526
|
+
```bash
|
|
527
|
+
export AGENTLOOP_API_URL=http://127.0.0.1:8000
|
|
528
|
+
export AGENTLOOP_API_KEY=al_xxx
|
|
529
|
+
uv run python examples/upload_trace_demo.py
|
|
530
|
+
```
|
|
531
|
+
|
|
532
|
+
## LangGraph usage
|
|
533
|
+
|
|
534
|
+
### Decorate individual nodes
|
|
535
|
+
|
|
536
|
+
```python
|
|
537
|
+
from agentloop import trace_agent
|
|
538
|
+
from agentloop.integrations.langgraph import trace_node
|
|
539
|
+
|
|
540
|
+
@trace_node("retrieve_sources")
|
|
541
|
+
def retrieve_sources(state):
|
|
542
|
+
return state
|
|
543
|
+
|
|
544
|
+
with trace_agent("research_graph") as trace:
|
|
545
|
+
retrieve_sources({})
|
|
546
|
+
|
|
547
|
+
trace.export_json("runs/research_graph.json")
|
|
548
|
+
```
|
|
549
|
+
|
|
550
|
+
### Auto-instrument a StateGraph builder
|
|
551
|
+
|
|
552
|
+
```python
|
|
553
|
+
from agentloop.integrations.langgraph import instrument_state_graph, trace_runnable
|
|
554
|
+
|
|
555
|
+
builder = StateGraph(State)
|
|
556
|
+
instrument_state_graph(builder)
|
|
557
|
+
|
|
558
|
+
builder.add_node("retrieve", retrieve)
|
|
559
|
+
builder.add_node("synthesize", synthesize)
|
|
560
|
+
|
|
561
|
+
app = trace_runnable(builder.compile(), name="research_graph")
|
|
562
|
+
app.invoke({"question": "What is agent profiling?"})
|
|
563
|
+
app.export_last_trace("runs/research_graph.json")
|
|
564
|
+
```
|
|
565
|
+
|
|
566
|
+
This integration is dependency-free. AgentLoop does not import LangGraph directly; it wraps StateGraph-like builders and compiled runnable-style objects.
|
|
567
|
+
|
|
568
|
+
## Project direction
|
|
569
|
+
|
|
570
|
+
Profiler -> Execution graph -> Optimization findings -> Replay proof -> Reliable deployment
|
|
571
|
+
|
|
572
|
+
The core scope is cost, latency, and reliability engineering for multi-step
|
|
573
|
+
agent workflows. AgentLoop remains local-first and self-hostable while keeping
|
|
574
|
+
its trace, analysis, and replay formats useful in CI and deployed environments.
|
|
575
|
+
|
|
576
|
+
See the contributor-facing [roadmap](docs/ROADMAP.md) for current capabilities,
|
|
577
|
+
priorities, and non-goals.
|
|
578
|
+
|
|
579
|
+
## Community
|
|
580
|
+
|
|
581
|
+
- Read [CONTRIBUTING.md](CONTRIBUTING.md) before proposing or submitting a
|
|
582
|
+
change.
|
|
583
|
+
- Use the structured [issue forms](https://github.com/dipeshbabu/agentloop/issues/new/choose)
|
|
584
|
+
for bugs, feature requests, and usage questions.
|
|
585
|
+
- Follow [SUPPORT.md](SUPPORT.md) for usage help and
|
|
586
|
+
[SECURITY.md](SECURITY.md) for private vulnerability reports.
|
|
587
|
+
- Project decisions follow [GOVERNANCE.md](GOVERNANCE.md), and all participation
|
|
588
|
+
is covered by the [Code of Conduct](CODE_OF_CONDUCT.md).
|
|
589
|
+
- Repository owners should complete the
|
|
590
|
+
[open-source launch checklist](docs/OPEN_SOURCE_CHECKLIST.md) before announcing
|
|
591
|
+
the public launch.
|
|
592
|
+
|
|
593
|
+
## License
|
|
594
|
+
|
|
595
|
+
Copyright 2026 Dipesh Babu and AgentLoop contributors.
|
|
596
|
+
|
|
597
|
+
AgentLoop is licensed under the [Apache License 2.0](LICENSE). Dependencies keep
|
|
598
|
+
their own terms; see the [third-party software inventory](THIRD_PARTY_LICENSES.md).
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
agentloop/__init__.py,sha256=zkSiIMBvx3HHKEtKvQ7gYkMUNrnuiaVt_42GpIWDTyM,1658
|
|
2
|
+
agentloop/audit.py,sha256=lDUwvv9CYb7F68jRsJGxv036Ju4GYGwEvj3fizaM-YA,1208
|
|
3
|
+
agentloop/autoinstrument.py,sha256=utf-Fw-N_mIhvPofKb8SIATTmVa5-GD3qNVqCw7eJU0,3724
|
|
4
|
+
agentloop/ci.py,sha256=fP87Whk7kLejG5hIstkVRhNu1EBUl4ukaPL4-6_QonQ,4883
|
|
5
|
+
agentloop/cli.py,sha256=40YsWGiGW5EE6YSzqZOibaNYkutBEWa38CdhiUKu9ik,33914
|
|
6
|
+
agentloop/client.py,sha256=LD2e5F-R5Sj3A3yAeVZsqwwkxJ9wDZHl63-YCjGJBnM,6618
|
|
7
|
+
agentloop/config.py,sha256=2eK0sALcKNq5SSDNkn7bB4hSe6MiBtfyJwNT0qR9rLE,1500
|
|
8
|
+
agentloop/costs.py,sha256=8fqcjdz6u4qwhkYhqxzPBAOpWggP_FUnaWYC6pMvh3o,589
|
|
9
|
+
agentloop/decorators.py,sha256=y3cYFpE1Uz6QqY6vWQNbAxe13K30ZRCDRVL97F85PEE,4832
|
|
10
|
+
agentloop/demo.py,sha256=RMExEo-hu1Ae04jqpA3I3fTYsamvynBD1o8Cr93wqo0,9438
|
|
11
|
+
agentloop/doctor.py,sha256=41fs7XzXQD53RydapnsvEp2qY5W2szmQBCPeO6GMVgs,11376
|
|
12
|
+
agentloop/events.py,sha256=Vy1FIFALH-WxxCTYBP8FeTEwTYrkrlULBARWB4Wc1NY,1115
|
|
13
|
+
agentloop/exporters.py,sha256=rE0kOsnrjo1m5LtbyouhHtK5KdL7DxTWcREJHf4rlCI,2147
|
|
14
|
+
agentloop/findings.py,sha256=HKK6Tl9cJknVuRAzN1zzdicrKxrssXhLoA-Ytj8K5hY,8842
|
|
15
|
+
agentloop/graph.py,sha256=VIIazUB8zQTjh75r2KdvrJUbVX-uJ_i7E1f2KfnUQjY,11508
|
|
16
|
+
agentloop/issues.py,sha256=0u75rRBXhBFxCQPPYZE6yWQkaZRt5f04rCuj0nMId5Y,3425
|
|
17
|
+
agentloop/markdown.py,sha256=GSqIJpXdlofzEiVK_MY_2PmXEmKFZx6uiFjl-0XyQHQ,3124
|
|
18
|
+
agentloop/metrics.py,sha256=nKHZL0BAWJ0b7J9iougG1557weg4csVdIXUnwd8bPNU,4944
|
|
19
|
+
agentloop/optimizer.py,sha256=tlfHzvEfsPtuXM0OcqoRGc4zjkWyBKzVEmq7I-ZPlGc,13926
|
|
20
|
+
agentloop/otel.py,sha256=xhSKkbaAoYbdAeHuQeadZ0WzSO_-zC0czgOxd30-ahI,10852
|
|
21
|
+
agentloop/otel_ids.py,sha256=8DNmcuKglXS1n1dgS9Z5vxkmpX3j59t60m-hPZn0EJQ,3347
|
|
22
|
+
agentloop/patches.py,sha256=atJht_5KZurLvw1ay7GpIdQSYJn6JtO-QvVbbwqCefs,24054
|
|
23
|
+
agentloop/plan_export.py,sha256=Hqd2bgKqLIwH0te8tx3HkbsZv-s_qhprenq_Poz2vmo,2839
|
|
24
|
+
agentloop/quality.py,sha256=5YYwZYjXn5Zr7mexSMtd3B81_vF97R6s2tQxMPL87Cg,8453
|
|
25
|
+
agentloop/replay.py,sha256=-YpISDUnA2oc0QGXpdiy_S0m28iTudJLLDO7mff7liM,16501
|
|
26
|
+
agentloop/runtime.py,sha256=rJYcNUwPWNemMr1hgpTFw_u4NqwEA02QAM6RzMyDR94,7621
|
|
27
|
+
agentloop/savings.py,sha256=jK__1zh9gYEvvHq5VYUXY-JADOosGqm-Z2jmPPk-_i0,5024
|
|
28
|
+
agentloop/server.py,sha256=BX6Wo-nwXoKp7K9GvA_r2rsNfY38mIkaYCnjxo_NlqU,9542
|
|
29
|
+
agentloop/store.py,sha256=ovZMvwJ_k5ATUZBt8-0a7-5_Z7e9ATu6QE3Xz4gkjZE,37364
|
|
30
|
+
agentloop/timing.py,sha256=Z7Caj1wdDxHNbcLdN1Mzd15KmhOjqm3glJJ7yd3SY9M,4401
|
|
31
|
+
agentloop/tracer.py,sha256=0gfucZjGAcccwyzMt8Y9Hfc8PUr0E66pbe9JO4dOITc,11449
|
|
32
|
+
agentloop/value.py,sha256=y9edxtRe-GS0vo8yEvqKMDoXUHq8msTOalQsLzI42SM,9944
|
|
33
|
+
agentloop/version.py,sha256=JZzxK5jFrwguJ3WpKyFiO5TBQkTocX2MsRYB64Dcrg8,58
|
|
34
|
+
agentloop/integrations/__init__.py,sha256=OViWhFa_N2V5FRB1bRqi1T9ErHAwzQqDZAoQVxqfrew,59
|
|
35
|
+
agentloop/integrations/crewai.py,sha256=IvsqI6yhgQMt2sQJr2aiz6XWZSqfMxyXT-7zpvNLe_k,4829
|
|
36
|
+
agentloop/integrations/langgraph.py,sha256=PLP_I2-qJ1DuG_dZIniVgsYmYHHPVrdMK9aVQQWYE3Q,5078
|
|
37
|
+
agentloop/integrations/openai.py,sha256=X6bFmB0UbRYo_doYKW_fftbgQ5fFmyHFQS3HBUefFiY,13017
|
|
38
|
+
agentloop/integrations/openai_agents.py,sha256=doDZGTgC45poE7rS-mTLly7ciDhPBNtBJRd6A3Gdl2E,8497
|
|
39
|
+
agentloop/integrations/sdk.py,sha256=VxsNnmKNYBdQ52Ehg55t7N1obei2xFMs4wm76MhCAUM,2765
|
|
40
|
+
agentloop/integrations/vercel_ai.py,sha256=QaCOTmwsom_1LlTVkI5FJ_eVLuC_GYt5r1IB7z7HPps,3029
|
|
41
|
+
agentloop_profiler-0.4.0.dist-info/licenses/LICENSE,sha256=N_a312brZ5HtuJIRERpI4-Ev9csGV4KZAZuwCf1b5Ew,11358
|
|
42
|
+
agentloop_profiler-0.4.0.dist-info/licenses/THIRD_PARTY_LICENSES.md,sha256=VXFATYOTUuMFE7mXSvlwrSqoZ927RJbKCUxUl590flA,2752
|
|
43
|
+
dashboard/__init__.py,sha256=svb9mKRvua4fZ6UewPYtBWz-3ZvveXuns7G0nGaaa0Y,49
|
|
44
|
+
dashboard/app.py,sha256=sRH_wg2qvmByQi6bzos1fk44XrakJpvZkzKnQ-LtKGs,29170
|
|
45
|
+
dashboard/value_view.py,sha256=euKr_jLuxnZBI6t9DvFpz60VPBkS5McwRoDN0VkooNA,3383
|
|
46
|
+
agentloop_profiler-0.4.0.dist-info/METADATA,sha256=gKr5ULIkBeJJIe_6fcMcDy5kUGezRJCjwteI-y8KvVk,24048
|
|
47
|
+
agentloop_profiler-0.4.0.dist-info/WHEEL,sha256=K260EYznzXsJYBQGqmI8VTxEdiZYNvDZwW9cBh9-_MA,91
|
|
48
|
+
agentloop_profiler-0.4.0.dist-info/entry_points.txt,sha256=h6GExkAPICtPhcYZkD_9CNLT8obw15KcaiwzxQTLb1Q,48
|
|
49
|
+
agentloop_profiler-0.4.0.dist-info/top_level.txt,sha256=vuLaOtUWmmxaqHEUp_E-mC1XuJSloLRjuFhQAAGFeZo,20
|
|
50
|
+
agentloop_profiler-0.4.0.dist-info/RECORD,,
|