agentloop-profiler 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (166) hide show
  1. agentloop/__init__.py +136 -0
  2. agentloop/ablation_protocol.py +229 -0
  3. agentloop/ablations.py +564 -0
  4. agentloop/aggregate_cli.py +88 -0
  5. agentloop/aggregate_stats.py +175 -0
  6. agentloop/aggregates.py +532 -0
  7. agentloop/audit.py +39 -0
  8. agentloop/autoinstrument.py +145 -0
  9. agentloop/batch_experiments.py +295 -0
  10. agentloop/batching.py +95 -0
  11. agentloop/budget_types.py +335 -0
  12. agentloop/budgets.py +539 -0
  13. agentloop/calibration.py +872 -0
  14. agentloop/checkpoint_state.py +306 -0
  15. agentloop/checkpoint_store.py +86 -0
  16. agentloop/ci.py +202 -0
  17. agentloop/cli.py +1314 -0
  18. agentloop/client.py +228 -0
  19. agentloop/completion.py +345 -0
  20. agentloop/completion_types.py +199 -0
  21. agentloop/config.py +50 -0
  22. agentloop/context_controls.py +443 -0
  23. agentloop/context_experiments.py +337 -0
  24. agentloop/context_types.py +216 -0
  25. agentloop/costs.py +667 -0
  26. agentloop/cursor_validation.py +48 -0
  27. agentloop/decorators.py +445 -0
  28. agentloop/demo.py +300 -0
  29. agentloop/doctor.py +341 -0
  30. agentloop/drift.py +361 -0
  31. agentloop/drift_cli.py +147 -0
  32. agentloop/drift_types.py +110 -0
  33. agentloop/drift_windows.py +340 -0
  34. agentloop/entrypoint.py +201 -0
  35. agentloop/estimates.py +133 -0
  36. agentloop/events.py +75 -0
  37. agentloop/experiment_artifacts.py +80 -0
  38. agentloop/experiment_observations.py +169 -0
  39. agentloop/experiment_reports.py +545 -0
  40. agentloop/experiment_types.py +355 -0
  41. agentloop/experiments.py +392 -0
  42. agentloop/exporters.py +117 -0
  43. agentloop/finding_benchmark_types.py +131 -0
  44. agentloop/finding_benchmarks.py +424 -0
  45. agentloop/findings.py +419 -0
  46. agentloop/graph.py +464 -0
  47. agentloop/harness.py +825 -0
  48. agentloop/harness_bench/__init__.py +1 -0
  49. agentloop/harness_bench/cli.py +77 -0
  50. agentloop/harness_bench/drivers.py +96 -0
  51. agentloop/harness_bench/offline.py +507 -0
  52. agentloop/harness_evidence.py +585 -0
  53. agentloop/harness_usage.py +91 -0
  54. agentloop/html_report.py +860 -0
  55. agentloop/integrations/__init__.py +1 -0
  56. agentloop/integrations/crewai.py +144 -0
  57. agentloop/integrations/harbor/__init__.py +1 -0
  58. agentloop/integrations/harbor/atif.py +982 -0
  59. agentloop/integrations/harbor/atif_validation.py +329 -0
  60. agentloop/integrations/harbor/cli.py +207 -0
  61. agentloop/integrations/harbor/manifest.py +86 -0
  62. agentloop/integrations/harbor/trial_evidence.py +196 -0
  63. agentloop/integrations/harbor/trials.py +1109 -0
  64. agentloop/integrations/harbor/verifier.py +118 -0
  65. agentloop/integrations/langgraph.py +150 -0
  66. agentloop/integrations/langgraph_harness.py +466 -0
  67. agentloop/integrations/langgraph_recovery.py +366 -0
  68. agentloop/integrations/omnigent/__init__.py +1 -0
  69. agentloop/integrations/omnigent/cli.py +78 -0
  70. agentloop/integrations/omnigent/evidence.py +93 -0
  71. agentloop/integrations/omnigent/telemetry.py +422 -0
  72. agentloop/integrations/openai.py +415 -0
  73. agentloop/integrations/openai_agents.py +291 -0
  74. agentloop/integrations/sdk.py +82 -0
  75. agentloop/integrations/vercel_ai.py +83 -0
  76. agentloop/interoperability/__init__.py +5 -0
  77. agentloop/interoperability/artifacts.py +112 -0
  78. agentloop/interoperability/cli.py +59 -0
  79. agentloop/interoperability/cohort_evidence.py +175 -0
  80. agentloop/interoperability/cohorts.py +681 -0
  81. agentloop/interoperability/contracts.py +427 -0
  82. agentloop/interoperability/coordination.py +554 -0
  83. agentloop/interoperability/evidence.py +215 -0
  84. agentloop/interoperability/otlp_jsonl.py +775 -0
  85. agentloop/interoperability/otlp_validation.py +496 -0
  86. agentloop/interoperability/registry.py +561 -0
  87. agentloop/interoperability/validation.py +333 -0
  88. agentloop/intervention_service.py +110 -0
  89. agentloop/intervention_store.py +118 -0
  90. agentloop/interventions.py +190 -0
  91. agentloop/issues.py +90 -0
  92. agentloop/judgment_adapters.py +125 -0
  93. agentloop/judgment_benchmark_types.py +98 -0
  94. agentloop/judgment_benchmarks.py +606 -0
  95. agentloop/judgment_types.py +241 -0
  96. agentloop/judgment_views.py +38 -0
  97. agentloop/judgments.py +342 -0
  98. agentloop/loop_guards.py +274 -0
  99. agentloop/loop_types.py +66 -0
  100. agentloop/markdown.py +102 -0
  101. agentloop/metrics.py +342 -0
  102. agentloop/migrations.py +303 -0
  103. agentloop/model_routing.py +470 -0
  104. agentloop/onboarding.py +315 -0
  105. agentloop/operations.py +44 -0
  106. agentloop/optimizer.py +199 -0
  107. agentloop/otel.py +715 -0
  108. agentloop/otel_ids.py +81 -0
  109. agentloop/otel_semantics.py +134 -0
  110. agentloop/parallelism.py +141 -0
  111. agentloop/patches.py +603 -0
  112. agentloop/plan_export.py +126 -0
  113. agentloop/policy_promotion.py +325 -0
  114. agentloop/promotion_evidence.py +304 -0
  115. agentloop/promotion_types.py +209 -0
  116. agentloop/quality.py +446 -0
  117. agentloop/ranking.py +529 -0
  118. agentloop/recovery_evidence.py +96 -0
  119. agentloop/replay.py +753 -0
  120. agentloop/retention.py +375 -0
  121. agentloop/retention_types.py +113 -0
  122. agentloop/reuse_experiments.py +247 -0
  123. agentloop/routing_checks.py +116 -0
  124. agentloop/routing_types.py +241 -0
  125. agentloop/rules.py +396 -0
  126. agentloop/runtime.py +286 -0
  127. agentloop/savings.py +158 -0
  128. agentloop/scheduling_plan.py +99 -0
  129. agentloop/scheduling_types.py +169 -0
  130. agentloop/schema.py +257 -0
  131. agentloop/semantic_rules.py +200 -0
  132. agentloop/semantic_waste.py +345 -0
  133. agentloop/semantic_waste_types.py +88 -0
  134. agentloop/server.py +494 -0
  135. agentloop/store.py +1429 -0
  136. agentloop/structural_experiments.py +431 -0
  137. agentloop/structural_types.py +113 -0
  138. agentloop/structured_quality.py +1020 -0
  139. agentloop/studies.py +629 -0
  140. agentloop/study_statistics.py +58 -0
  141. agentloop/substitution_evidence.py +199 -0
  142. agentloop/substitution_export.py +190 -0
  143. agentloop/substitution_reports.py +407 -0
  144. agentloop/substitution_types.py +284 -0
  145. agentloop/substitutions.py +267 -0
  146. agentloop/timing.py +142 -0
  147. agentloop/tokens.py +188 -0
  148. agentloop/tool_scheduling.py +410 -0
  149. agentloop/trace_sources.py +31 -0
  150. agentloop/tracer.py +494 -0
  151. agentloop/value.py +306 -0
  152. agentloop/version.py +3 -0
  153. agentloop/workflow_types.py +264 -0
  154. agentloop/workflows.py +284 -0
  155. agentloop_profiler-0.1.0.dist-info/METADATA +513 -0
  156. agentloop_profiler-0.1.0.dist-info/RECORD +166 -0
  157. agentloop_profiler-0.1.0.dist-info/WHEEL +5 -0
  158. agentloop_profiler-0.1.0.dist-info/entry_points.txt +2 -0
  159. agentloop_profiler-0.1.0.dist-info/licenses/LICENSE +201 -0
  160. agentloop_profiler-0.1.0.dist-info/licenses/THIRD_PARTY_LICENSES.md +101 -0
  161. agentloop_profiler-0.1.0.dist-info/top_level.txt +2 -0
  162. dashboard/__init__.py +5 -0
  163. dashboard/app.py +922 -0
  164. dashboard/streamlit_compat.py +63 -0
  165. dashboard/trace_upload.py +57 -0
  166. dashboard/value_view.py +105 -0
@@ -0,0 +1,340 @@
1
+ """Bounded cohort windows built from native traces or existing aggregate artifacts."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import copy
6
+ import hashlib
7
+ import json
8
+ from collections import Counter
9
+ from dataclasses import asdict
10
+
11
+ from agentloop.aggregate_stats import AggregateConfig, number, total
12
+ from agentloop.aggregates import TraceAggregate
13
+ from agentloop.drift_types import QualityMetric, WindowSpec, label, timestamp
14
+ from agentloop.onboarding import validate_capture
15
+ from agentloop.operations import operation_kind
16
+ from agentloop.retention import read_retention
17
+ from agentloop.rules import BUILTIN_RULES
18
+ from agentloop.workflow_types import stage_summary, workflow_summary
19
+
20
+ DRIFT_VERSION = "1.0"
21
+
22
+
23
+ def digest(value):
24
+ return hashlib.sha256(
25
+ json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False).encode()
26
+ ).hexdigest()
27
+
28
+
29
+ def seal(value):
30
+ result = copy.deepcopy(value)
31
+ result["sha256"] = digest(result)
32
+ return result
33
+
34
+
35
+ def verify(value, kind):
36
+ if (
37
+ not isinstance(value, dict)
38
+ or value.get("schema_version") != DRIFT_VERSION
39
+ or value.get("kind") != kind
40
+ ):
41
+ raise ValueError(f"unsupported {kind} artifact")
42
+ content = dict(value)
43
+ if content.pop("sha256", None) != digest(content):
44
+ raise ValueError(f"{kind} content hash mismatch")
45
+
46
+
47
+ def _mix():
48
+ return {"counts": Counter(), "overflow": 0, "missing": 0, "total": 0}
49
+
50
+
51
+ def _mix_add(mix, value):
52
+ mix["total"] += 1
53
+ if value is None:
54
+ mix["missing"] += 1
55
+ return
56
+ key = digest(value)
57
+ if key in mix["counts"] or len(mix["counts"]) < 128:
58
+ mix["counts"][key] += 1
59
+ else:
60
+ mix["overflow"] += 1
61
+
62
+
63
+ class WindowBuilder:
64
+ def __init__(
65
+ self,
66
+ spec: WindowSpec,
67
+ *,
68
+ quality_metrics: tuple[QualityMetric, ...] = (),
69
+ config: AggregateConfig | None = None,
70
+ include_findings=False,
71
+ ):
72
+ if not isinstance(spec, WindowSpec):
73
+ raise TypeError("spec must be WindowSpec")
74
+ if (
75
+ not isinstance(quality_metrics, tuple)
76
+ or len(quality_metrics) > 16
77
+ or any(not isinstance(metric, QualityMetric) for metric in quality_metrics)
78
+ ):
79
+ raise ValueError("quality_metrics must be a tuple of at most 16 declarations")
80
+ if len({metric.name for metric in quality_metrics}) != len(quality_metrics):
81
+ raise ValueError("quality metric names must be unique")
82
+ if type(include_findings) is not bool:
83
+ raise ValueError("include_findings must be boolean")
84
+ self.spec = spec
85
+ self.quality_metrics = tuple(sorted(quality_metrics, key=lambda metric: metric.name))
86
+ self.config = AggregateConfig() if config is None else config
87
+ if not isinstance(self.config, AggregateConfig):
88
+ raise TypeError("config must be AggregateConfig")
89
+ self.include_findings = include_findings
90
+ self._cohorts = {}
91
+ self._chain = digest([])
92
+ self._sources = 0
93
+
94
+ def _cohort(self, cohort):
95
+ label(cohort, "cohort")
96
+ if cohort not in self._cohorts and len(self._cohorts) >= 32:
97
+ raise ValueError("a drift window supports at most 32 explicit cohorts")
98
+ if cohort in self._cohorts:
99
+ return copy.deepcopy(self._cohorts[cohort])
100
+ return {
101
+ "aggregate": TraceAggregate(self.config),
102
+ "flags": {
103
+ key: {"known": 0, "true": 0}
104
+ for key in ("timeout", "abstention", "finding", "instrumentation")
105
+ },
106
+ "quality": {metric.name: {"count": 0, "sum": 0.0} for metric in self.quality_metrics},
107
+ "mixes": {key: _mix() for key in ("model", "stage", "outcome", "decision_outcome")},
108
+ "individual_evidence_missing_runs": 0,
109
+ "producer_markers": {"synthetic": 0, "declared_real": 0, "unspecified": 0},
110
+ "time_membership": {
111
+ "native_ended_at_verified_runs": 0,
112
+ "host_assigned_aggregate_runs": 0,
113
+ },
114
+ }
115
+
116
+ def add(self, trace, *, cohort="all", timeout=None, abstention=None, quality=None):
117
+ ended = timestamp(trace.ended_at)
118
+ if not timestamp(self.spec.started_at) <= ended < timestamp(self.spec.ended_at):
119
+ raise ValueError("trace completion is outside the declared window")
120
+ for value in (timeout, abstention):
121
+ if value is not None and type(value) is not bool:
122
+ raise ValueError("timeout/abstention must be boolean or None")
123
+ quality = {} if quality is None else quality
124
+ if not isinstance(quality, dict) or not set(quality) <= {
125
+ metric.name for metric in self.quality_metrics
126
+ }:
127
+ raise ValueError("quality values must match declared metrics")
128
+ for value in quality.values():
129
+ if value is not None and (
130
+ type(value) not in {int, float} or not 0 <= number(value) <= 1
131
+ ):
132
+ raise ValueError("quality values must be in [0,1] or None")
133
+ owned = self._cohort(cohort)
134
+ owned["aggregate"].add(trace)
135
+ owned["time_membership"]["native_ended_at_verified_runs"] += 1
136
+ synthetic = trace.metadata.get("synthetic")
137
+ owned["producer_markers"][
138
+ "synthetic"
139
+ if synthetic is True
140
+ else "declared_real"
141
+ if synthetic is False
142
+ else "unspecified"
143
+ ] += 1
144
+ retained = read_retention(trace)
145
+ complete = retained is None or retained["complete_evidence"]
146
+ for key, value in (("timeout", timeout), ("abstention", abstention)):
147
+ if value is not None:
148
+ owned["flags"][key]["known"] += 1
149
+ owned["flags"][key]["true"] += value
150
+ for key, value in quality.items():
151
+ if value is not None:
152
+ owned["quality"][key]["count"] += 1
153
+ owned["quality"][key]["sum"] = total(owned["quality"][key]["sum"], value)
154
+ owned["flags"]["instrumentation"]["known"] += 1
155
+ if complete:
156
+ validation = validate_capture(trace)
157
+ missing_codes = {
158
+ "no_execution_spans",
159
+ "unsupported_operation",
160
+ "usage_not_exact",
161
+ "model_identity_missing",
162
+ "trace_boundary_missing",
163
+ "span_timing_unavailable",
164
+ "span_outside_boundary",
165
+ "retained_evidence_incomplete",
166
+ }
167
+ owned["flags"]["instrumentation"]["true"] += not any(
168
+ check["severity"] == "error" or check["code"] in missing_codes
169
+ for check in validation["checks"]
170
+ )
171
+ for event in trace.events:
172
+ if operation_kind(event) == "model":
173
+ _mix_add(owned["mixes"]["model"], event.model)
174
+ stage = stage_summary(event)
175
+ _mix_add(
176
+ owned["mixes"]["stage"],
177
+ [stage.get("stage_id"), stage.get("version")]
178
+ if stage and stage.get("stage_id")
179
+ else None,
180
+ )
181
+ if operation_kind(event) in {"classifier", "rule"}:
182
+ _mix_add(
183
+ owned["mixes"]["decision_outcome"], stage.get("outcome") if stage else None
184
+ )
185
+ execution = workflow_summary(trace.metadata)
186
+ _mix_add(owned["mixes"]["outcome"], execution.get("outcome") if execution else None)
187
+ if self.include_findings and len(trace.events) <= 2000:
188
+ report = trace.report()
189
+ if report.get("analysis_complete"):
190
+ owned["flags"]["finding"]["known"] += 1
191
+ owned["flags"]["finding"]["true"] += bool(report["finding_candidates"])
192
+ else:
193
+ owned["individual_evidence_missing_runs"] += 1
194
+ source_hash = digest(trace.to_dict())
195
+ chain = digest([self._chain, source_hash, cohort, timeout, abstention, quality])
196
+ # All validation/analysis/hash work precedes mutation of builder state.
197
+ self._cohorts[cohort] = owned
198
+ self._chain, self._sources = chain, self._sources + 1
199
+
200
+ def add_aggregate(self, aggregate: TraceAggregate, *, cohort="all", time_assignment_ref):
201
+ """Existing aggregates contribute original metrics, with missing per-run extras explicit."""
202
+ if not isinstance(aggregate, TraceAggregate):
203
+ raise TypeError("aggregate must be TraceAggregate")
204
+ label(time_assignment_ref, "time_assignment_ref")
205
+ owned = self._cohort(cohort)
206
+ owned["aggregate"].merge(aggregate)
207
+ report = aggregate.to_dict()
208
+ owned["individual_evidence_missing_runs"] += report["counts"]["runs"]
209
+ owned["time_membership"]["host_assigned_aggregate_runs"] += report["counts"]["runs"]
210
+ owned["producer_markers"]["unspecified"] += report["counts"]["runs"]
211
+ chain = digest([self._chain, report["sha256"], cohort, time_assignment_ref])
212
+ self._cohorts[cohort] = owned
213
+ self._chain, self._sources = chain, self._sources + 1
214
+
215
+ def to_dict(self):
216
+ cohorts = copy.deepcopy(self._cohorts)
217
+ for values in cohorts.values():
218
+ values["aggregate"] = values["aggregate"].to_dict()
219
+ for mix in values["mixes"].values():
220
+ mix["counts"] = dict(mix["counts"])
221
+ return seal(
222
+ {
223
+ "schema_version": DRIFT_VERSION,
224
+ "kind": "agentloop.drift_window",
225
+ "spec": asdict(self.spec),
226
+ "quality_metrics": [asdict(metric) for metric in self.quality_metrics],
227
+ "settings": {
228
+ "include_findings": self.include_findings,
229
+ "rule_versions": [[rule.rule_id, rule.version] for rule in BUILTIN_RULES]
230
+ if self.include_findings
231
+ else None,
232
+ },
233
+ "cohorts": cohorts,
234
+ "source_count": self._sources,
235
+ "source_chain_sha256": self._chain,
236
+ "scope": "Supplied disjoint records only; cohort labels and quality/timeout/abstention annotations are host observations.",
237
+ }
238
+ )
239
+
240
+
241
+ def validate_window(value):
242
+ verify(value, "agentloop.drift_window")
243
+ WindowSpec(**value["spec"])
244
+ metrics = tuple(QualityMetric(**item) for item in value["quality_metrics"])
245
+ if (
246
+ type(value["source_count"]) is not int
247
+ or value["source_count"] < 0
248
+ or not isinstance(value["source_chain_sha256"], str)
249
+ or len(value["source_chain_sha256"]) != 64
250
+ or any(char not in "0123456789abcdef" for char in value["source_chain_sha256"])
251
+ ):
252
+ raise ValueError("invalid source chain")
253
+ settings = value["settings"]
254
+ if (
255
+ not isinstance(settings, dict)
256
+ or set(settings) != {"include_findings", "rule_versions"}
257
+ or type(settings["include_findings"]) is not bool
258
+ ):
259
+ raise ValueError("invalid window analysis settings")
260
+ if settings["include_findings"]:
261
+ if not isinstance(settings["rule_versions"], list) or len(settings["rule_versions"]) > 128:
262
+ raise ValueError("invalid finding rule versions")
263
+ for entry in settings["rule_versions"]:
264
+ if not isinstance(entry, list) or len(entry) != 2:
265
+ raise ValueError("invalid finding rule version entry")
266
+ for field in entry:
267
+ label(field, "finding rule version")
268
+ elif settings["rule_versions"] is not None:
269
+ raise ValueError("unexpected finding rule versions")
270
+ if (
271
+ len(metrics) > 16
272
+ or len({metric.name for metric in metrics}) != len(metrics)
273
+ or not isinstance(value["cohorts"], dict)
274
+ or len(value["cohorts"]) > 32
275
+ ):
276
+ raise ValueError("invalid drift window dimensions")
277
+ for cohort, data in value["cohorts"].items():
278
+ label(cohort, "cohort")
279
+ report = TraceAggregate.from_dict(data["aggregate"]).to_dict()
280
+ runs = report["counts"]["runs"]
281
+ markers = data["producer_markers"]
282
+ if (
283
+ set(markers) != {"synthetic", "declared_real", "unspecified"}
284
+ or any(type(count) is not int or count < 0 for count in markers.values())
285
+ or sum(markers.values()) != runs
286
+ ):
287
+ raise ValueError("invalid producer marker counts")
288
+ membership = data["time_membership"]
289
+ if (
290
+ set(membership) != {"native_ended_at_verified_runs", "host_assigned_aggregate_runs"}
291
+ or any(type(count) is not int or count < 0 for count in membership.values())
292
+ or sum(membership.values()) != runs
293
+ ):
294
+ raise ValueError("invalid time membership counts")
295
+ if (
296
+ set(data["quality"]) != {metric.name for metric in metrics}
297
+ or set(data["flags"]) != {"timeout", "abstention", "finding", "instrumentation"}
298
+ or set(data["mixes"]) != {"model", "stage", "outcome", "decision_outcome"}
299
+ ):
300
+ raise ValueError("invalid window metric fields")
301
+ for flag in data["flags"].values():
302
+ if (
303
+ any(type(flag[key]) is not int for key in ("known", "true"))
304
+ or not 0 <= flag["true"] <= flag["known"] <= runs
305
+ ):
306
+ raise ValueError("invalid window flag denominator")
307
+ for quality in data["quality"].values():
308
+ if (
309
+ type(quality["count"]) is not int
310
+ or not 0 <= quality["count"] <= runs
311
+ or not 0 <= number(quality["sum"]) <= quality["count"]
312
+ ):
313
+ raise ValueError("invalid window quality denominator")
314
+ for mix in data["mixes"].values():
315
+ counts = mix["counts"]
316
+ if (
317
+ not isinstance(counts, dict)
318
+ or len(counts) > 128
319
+ or any(
320
+ not isinstance(key, str)
321
+ or len(key) != 64
322
+ or any(char not in "0123456789abcdef" for char in key)
323
+ for key in counts
324
+ )
325
+ ):
326
+ raise ValueError("invalid window mix keys")
327
+ if (
328
+ any(
329
+ type(item) is not int or item < 0
330
+ for item in [*counts.values(), mix["missing"], mix["overflow"], mix["total"]]
331
+ )
332
+ or sum(counts.values()) + mix["missing"] + mix["overflow"] != mix["total"]
333
+ ):
334
+ raise ValueError("invalid window mix denominator")
335
+ if (
336
+ type(data["individual_evidence_missing_runs"]) is not int
337
+ or not 0 <= data["individual_evidence_missing_runs"] <= runs
338
+ ):
339
+ raise ValueError("invalid individual evidence count")
340
+ return copy.deepcopy(value)
@@ -0,0 +1,201 @@
1
+ from __future__ import annotations
2
+
3
+ from pathlib import Path
4
+
5
+ import typer
6
+ from rich.table import Table
7
+
8
+ from agentloop.cli import _build_quality_report_from_file, _load_trace, _write_json, app, console
9
+ from agentloop.events import AgentEvent
10
+ from agentloop.findings import build_diagnosis
11
+ from agentloop.html_report import analysis_to_html
12
+ from agentloop.optimizer import build_optimization_plan
13
+ from agentloop.ranking import RankingSort
14
+ from agentloop.replay import ReplayGates, build_replay_report
15
+ from agentloop.timing import format_duration_ms
16
+ from agentloop.tracer import AgentTrace
17
+
18
+
19
+ def _quickstart_trace() -> AgentTrace:
20
+ """Build a deterministic synthetic trace with several obvious bottlenecks."""
21
+ run_id = "run_agentloop_quickstart"
22
+ trace = AgentTrace(
23
+ name="agentloop_quickstart",
24
+ run_id=run_id,
25
+ metadata={"synthetic": True, "source": "agentloop_quickstart"},
26
+ started_at="2026-01-01T00:00:00+00:00",
27
+ ended_at="2026-01-01T00:00:01+00:00",
28
+ elapsed_ms=1000.0,
29
+ )
30
+ events = [
31
+ ("tool_call", "retrieve", 0.0, 180.0, None, 0, 0),
32
+ ("tool_call", "retrieve", 180.0, 360.0, None, 0, 0),
33
+ ("tool_call", "retrieve", 360.0, 540.0, None, 0, 0),
34
+ ("model_call", "summarize", 540.0, 650.0, "gpt-4.1", 700, 80),
35
+ ("model_call", "summarize", 650.0, 760.0, "gpt-4.1", 700, 80),
36
+ ("model_call", "summarize", 760.0, 870.0, "gpt-4.1", 700, 80),
37
+ ("retry", "repair_json", 870.0, 930.0, None, 0, 0),
38
+ ]
39
+ for index, (
40
+ event_type,
41
+ name,
42
+ start_ms,
43
+ end_ms,
44
+ model,
45
+ input_tokens,
46
+ output_tokens,
47
+ ) in enumerate(events, start=1):
48
+ start_seconds = start_ms / 1000
49
+ end_seconds = end_ms / 1000
50
+ trace.add_event(
51
+ AgentEvent(
52
+ event_id=f"evt_quickstart_{index:02d}",
53
+ run_id=run_id,
54
+ event_type=event_type,
55
+ name=name,
56
+ started_at=f"2026-01-01T00:00:{start_seconds:06.3f}+00:00",
57
+ ended_at=f"2026-01-01T00:00:{end_seconds:06.3f}+00:00",
58
+ duration_ms=end_ms - start_ms,
59
+ model=model,
60
+ input_tokens=input_tokens,
61
+ output_tokens=output_tokens,
62
+ input_text=("stable research instruction " * 30 if model else None),
63
+ )
64
+ )
65
+ return trace
66
+
67
+
68
+ def _print_analysis(trace: AgentTrace, diagnosis: dict, plan: dict) -> None:
69
+ report = trace.report()
70
+ summary = diagnosis["summary"]
71
+ table = Table(title=f"AgentLoop Analysis: {trace.name}")
72
+ table.add_column("Metric")
73
+ table.add_column("Value")
74
+ table.add_row("Runtime", format_duration_ms(report["total_runtime_ms"], precision=3))
75
+ if report.get("external_evidence"):
76
+ source = report["external_evidence"]["source"]
77
+ table.add_row("External source", str(source.get("system", "unknown")))
78
+ table.add_row("Execution status", report["external_evidence"]["execution_status"])
79
+ table.add_row("Timing coverage", report["timing_status"])
80
+ table.add_row("Findings", str(summary["finding_count"]))
81
+ table.add_row("High severity", str(summary["high_severity_count"]))
82
+ table.add_row("Patchable", str(summary["patchable_count"]))
83
+ table.add_row("Retries", str(report["retry_count"]))
84
+ table.add_row("Input tokens", str(report["input_tokens"]))
85
+ console.print(table)
86
+
87
+ if report.get("rule_errors"):
88
+ console.print(
89
+ "Analysis incomplete; failed rules: "
90
+ + ", ".join(error["rule_id"] for error in report["rule_errors"]),
91
+ markup=False,
92
+ )
93
+
94
+ findings = diagnosis.get("findings", [])
95
+ if not findings:
96
+ console.print("No optimization findings detected.")
97
+ return
98
+ console.print("\nTop findings:")
99
+ for finding in findings[:5]:
100
+ console.print(
101
+ f"- [{finding['ranking']['status'].replace('_', ' ')}] {finding['title']} ({finding['confidence']} confidence)",
102
+ markup=False,
103
+ )
104
+ aggregation = plan.get("savings_aggregation", {})
105
+ if aggregation.get("selection_optimal") is False:
106
+ console.print("Savings selection includes an approximate large-component fallback.")
107
+
108
+
109
+ def _analysis_payload(trace: AgentTrace, *, sort_by="priority") -> dict:
110
+ report = trace.report()
111
+ diagnosis = build_diagnosis(trace, sort_by=sort_by)
112
+ plan = build_optimization_plan(trace, report=report)
113
+ return {
114
+ "trace": trace.to_dict(),
115
+ "report": report,
116
+ "diagnosis": diagnosis,
117
+ "optimization": plan,
118
+ }
119
+
120
+
121
+ @app.command("quickstart")
122
+ def quickstart_command(
123
+ out: Path = typer.Option(
124
+ Path("runs/agentloop_quickstart.json"),
125
+ help="Where to write the deterministic synthetic trace.",
126
+ ),
127
+ json_out: Path | None = typer.Option(
128
+ None, help="Optional path for the combined analysis JSON."
129
+ ),
130
+ ) -> None:
131
+ """Generate one offline trace and show useful findings with no account or API key."""
132
+ trace = _quickstart_trace()
133
+ trace.export_json(out)
134
+ payload = _analysis_payload(trace)
135
+ _print_analysis(trace, payload["diagnosis"], payload["optimization"])
136
+ console.print(f"\nSynthetic trace: {out}")
137
+ console.print(f"Next: agentloop analyze {out}")
138
+ if json_out is not None:
139
+ _write_json(json_out, payload)
140
+ console.print(f"Analysis JSON: {json_out}")
141
+
142
+
143
+ @app.command("analyze")
144
+ def analyze_command(
145
+ path: Path,
146
+ json_out: Path | None = typer.Option(
147
+ None, help="Optional path for report, diagnosis, and optimization JSON."
148
+ ),
149
+ html_out: Path | None = typer.Option(
150
+ None, "--html", help="Write a single offline HTML report."
151
+ ),
152
+ baseline: Path | None = typer.Option(
153
+ None, help="Optional baseline trace; the positional trace is the candidate."
154
+ ),
155
+ quality_fixtures: Path | None = typer.Option(
156
+ None, help="Quality fixtures for the optional baseline comparison."
157
+ ),
158
+ min_quality_score: float | None = typer.Option(None, min=0, max=1),
159
+ include_content: bool = typer.Option(
160
+ False, help="Include raw event text and full event metadata in HTML."
161
+ ),
162
+ sort_by: RankingSort = typer.Option(
163
+ RankingSort.PRIORITY, help="Order findings by an explicit ranking dimension."
164
+ ),
165
+ ) -> None:
166
+ """Analyze one existing AgentLoop trace in a single command."""
167
+ trace = _load_trace(path, param_hint="path")
168
+ if (quality_fixtures is not None or min_quality_score is not None) and baseline is None:
169
+ raise typer.BadParameter("quality comparison options require --baseline")
170
+ if include_content and html_out is None:
171
+ raise typer.BadParameter("--include-content requires --html")
172
+ payload = _analysis_payload(trace, sort_by=sort_by.value)
173
+ if baseline is not None:
174
+ baseline_trace = _load_trace(baseline, param_hint="--baseline")
175
+ quality = (
176
+ _build_quality_report_from_file(
177
+ quality_fixtures,
178
+ param_hint="--quality-fixtures",
179
+ baseline_trace=baseline_trace,
180
+ candidate_trace=trace,
181
+ min_score=min_quality_score,
182
+ )
183
+ if quality_fixtures is not None
184
+ else None
185
+ )
186
+ payload["replay"] = build_replay_report(
187
+ baseline_trace,
188
+ trace,
189
+ gates=ReplayGates(min_quality_score=min_quality_score),
190
+ quality_report=quality,
191
+ )
192
+ _print_analysis(trace, payload["diagnosis"], payload["optimization"])
193
+ if json_out is not None:
194
+ _write_json(json_out, payload)
195
+ console.print(f"Wrote analysis JSON to {json_out}")
196
+ if html_out is not None:
197
+ html_out.parent.mkdir(parents=True, exist_ok=True)
198
+ html_out.write_text(
199
+ analysis_to_html(payload, include_content=include_content), encoding="utf-8"
200
+ )
201
+ console.print(f"Wrote HTML report to {html_out}")
agentloop/estimates.py ADDED
@@ -0,0 +1,133 @@
1
+ """Versioned, uncalibrated savings models and self-contained evidence snapshots."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ from dataclasses import dataclass
7
+ from typing import Any
8
+
9
+ from agentloop.costs import is_cost_evaluable
10
+ from agentloop.markdown import markdown_code_span, markdown_text
11
+
12
+
13
+ @dataclass(frozen=True)
14
+ class Estimator:
15
+ formula: str
16
+ assumptions: tuple[str, ...]
17
+ parameters: tuple[tuple[str, float], ...] = ()
18
+ version: str = "1.0"
19
+
20
+ def parameter(self, name: str) -> float:
21
+ return dict(self.parameters)[name]
22
+
23
+
24
+ ESTIMATORS = {
25
+ "parallelize_tools": Estimator(
26
+ "sum_duration_ms - max_duration_ms",
27
+ ("The formula applies assuming concurrent execution has negligible scheduling overhead.",),
28
+ ),
29
+ "cache_context": Estimator(
30
+ "current_cost_usd * min(max_cost_fraction, repeated_context_ratio)",
31
+ (
32
+ "Repeated context can be reused without changing required outputs.",
33
+ "The reusable share of total priced cost approximates the repeated context ratio.",
34
+ "The provider or application supports the proposed cache.",
35
+ ),
36
+ (("max_cost_fraction", 0.5),),
37
+ ),
38
+ "add_schema_validation": Estimator(
39
+ "sum_duration_ms * latency_fraction",
40
+ (
41
+ "Most observed retries are preventable with validation or repair.",
42
+ "Added validation overhead is smaller than avoided retry time.",
43
+ ),
44
+ (("latency_fraction", 0.8),),
45
+ ),
46
+ "batch_model_calls": Estimator(
47
+ "sum_duration_ms * latency_fraction",
48
+ (
49
+ "Repeated calls accept a shared batch without changing item-level outputs.",
50
+ "Batch size, token limits, and provider throughput permit the estimated reduction.",
51
+ ),
52
+ (("latency_fraction", 0.35),),
53
+ ),
54
+ "route_to_smaller_model": Estimator(
55
+ "sum_duration_ms * latency_fraction",
56
+ (
57
+ "A smaller model meets the task's quality requirements.",
58
+ "Its serving latency is lower under comparable load.",
59
+ ),
60
+ (("latency_fraction", 0.25),),
61
+ ),
62
+ "split_large_step": Estimator(
63
+ "sum_duration_ms * latency_fraction",
64
+ (
65
+ "Context can be compressed or split without losing necessary information.",
66
+ "Extra orchestration and model calls do not consume the estimated savings.",
67
+ ),
68
+ (("latency_fraction", 0.20),),
69
+ ),
70
+ "runaway_loop": Estimator(
71
+ "sum_duration_ms * latency_fraction",
72
+ (
73
+ "Some repeated steps are unnecessary for successful task completion.",
74
+ "A guard can stop them without prematurely terminating useful work.",
75
+ ),
76
+ (("latency_fraction", 0.30),),
77
+ ),
78
+ "tool_oscillation": Estimator(
79
+ "sum_duration_ms * latency_fraction",
80
+ (
81
+ "Some alternating tool calls repeat equivalent state transitions.",
82
+ "A state-change check can remove them without losing required transitions.",
83
+ ),
84
+ (("latency_fraction", 0.40),),
85
+ ),
86
+ }
87
+
88
+
89
+ def estimate_snapshot(candidate: Any, report: dict[str, Any], nodes: list[Any]) -> dict[str, Any]:
90
+ """Copy the estimator and its inputs; never reconstruct historical records on read."""
91
+ spec = ESTIMATORS[candidate.rule_id]
92
+ affected = set(candidate.affected_nodes or [])
93
+ durations = [node.duration_ms for node in nodes if node.node_id in affected]
94
+ cost_status = report.get("cost_status", "complete")
95
+ return {
96
+ "estimator_id": candidate.rule_id,
97
+ "estimator_version": spec.version,
98
+ "method": "heuristic",
99
+ "confidence": candidate.confidence,
100
+ "calibrated": False,
101
+ "formula": spec.formula,
102
+ "parameters": dict(spec.parameters),
103
+ "inputs": {
104
+ "sum_duration_ms": sum(durations),
105
+ "max_duration_ms": max(durations, default=0.0),
106
+ "span_count": len(durations),
107
+ "current_cost_usd": (
108
+ report.get("estimated_cost_usd", 0.0) if is_cost_evaluable(cost_status) else None
109
+ ),
110
+ "repeated_context_ratio": report.get("repeated_context_ratio", 0.0),
111
+ "cost_status": cost_status,
112
+ "token_status": report.get("token_status", "unspecified"),
113
+ },
114
+ "assumptions": list(dict.fromkeys([*(candidate.assumptions or []), *spec.assumptions])),
115
+ "unmodeled_metrics": (["latency"] if candidate.rule_id == "cache_context" else ["cost"]),
116
+ }
117
+
118
+
119
+ def estimate_markdown(estimate: dict[str, Any] | None) -> list[str]:
120
+ if not estimate:
121
+ return []
122
+ calibration = "calibrated" if estimate.get("calibrated") is True else "uncalibrated"
123
+ return [
124
+ f"- Estimator: {markdown_code_span(estimate['estimator_id'])} "
125
+ f"version {markdown_text(estimate['estimator_version'])}",
126
+ f"- Method: {markdown_text(estimate['method'])} ({calibration}; predicted savings)",
127
+ f"- Savings formula: {markdown_code_span(estimate['formula'])}",
128
+ "- Parameters: "
129
+ + markdown_code_span(json.dumps(estimate.get("parameters", {}), sort_keys=True)),
130
+ "- Assumptions: "
131
+ + "; ".join(markdown_text(value) for value in estimate.get("assumptions", [])),
132
+ "",
133
+ ]