agentloop-profiler 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agentloop/__init__.py +136 -0
- agentloop/ablation_protocol.py +229 -0
- agentloop/ablations.py +564 -0
- agentloop/aggregate_cli.py +88 -0
- agentloop/aggregate_stats.py +175 -0
- agentloop/aggregates.py +532 -0
- agentloop/audit.py +39 -0
- agentloop/autoinstrument.py +145 -0
- agentloop/batch_experiments.py +295 -0
- agentloop/batching.py +95 -0
- agentloop/budget_types.py +335 -0
- agentloop/budgets.py +539 -0
- agentloop/calibration.py +872 -0
- agentloop/checkpoint_state.py +306 -0
- agentloop/checkpoint_store.py +86 -0
- agentloop/ci.py +202 -0
- agentloop/cli.py +1314 -0
- agentloop/client.py +228 -0
- agentloop/completion.py +345 -0
- agentloop/completion_types.py +199 -0
- agentloop/config.py +50 -0
- agentloop/context_controls.py +443 -0
- agentloop/context_experiments.py +337 -0
- agentloop/context_types.py +216 -0
- agentloop/costs.py +667 -0
- agentloop/cursor_validation.py +48 -0
- agentloop/decorators.py +445 -0
- agentloop/demo.py +300 -0
- agentloop/doctor.py +341 -0
- agentloop/drift.py +361 -0
- agentloop/drift_cli.py +147 -0
- agentloop/drift_types.py +110 -0
- agentloop/drift_windows.py +340 -0
- agentloop/entrypoint.py +201 -0
- agentloop/estimates.py +133 -0
- agentloop/events.py +75 -0
- agentloop/experiment_artifacts.py +80 -0
- agentloop/experiment_observations.py +169 -0
- agentloop/experiment_reports.py +545 -0
- agentloop/experiment_types.py +355 -0
- agentloop/experiments.py +392 -0
- agentloop/exporters.py +117 -0
- agentloop/finding_benchmark_types.py +131 -0
- agentloop/finding_benchmarks.py +424 -0
- agentloop/findings.py +419 -0
- agentloop/graph.py +464 -0
- agentloop/harness.py +825 -0
- agentloop/harness_bench/__init__.py +1 -0
- agentloop/harness_bench/cli.py +77 -0
- agentloop/harness_bench/drivers.py +96 -0
- agentloop/harness_bench/offline.py +507 -0
- agentloop/harness_evidence.py +585 -0
- agentloop/harness_usage.py +91 -0
- agentloop/html_report.py +860 -0
- agentloop/integrations/__init__.py +1 -0
- agentloop/integrations/crewai.py +144 -0
- agentloop/integrations/harbor/__init__.py +1 -0
- agentloop/integrations/harbor/atif.py +982 -0
- agentloop/integrations/harbor/atif_validation.py +329 -0
- agentloop/integrations/harbor/cli.py +207 -0
- agentloop/integrations/harbor/manifest.py +86 -0
- agentloop/integrations/harbor/trial_evidence.py +196 -0
- agentloop/integrations/harbor/trials.py +1109 -0
- agentloop/integrations/harbor/verifier.py +118 -0
- agentloop/integrations/langgraph.py +150 -0
- agentloop/integrations/langgraph_harness.py +466 -0
- agentloop/integrations/langgraph_recovery.py +366 -0
- agentloop/integrations/omnigent/__init__.py +1 -0
- agentloop/integrations/omnigent/cli.py +78 -0
- agentloop/integrations/omnigent/evidence.py +93 -0
- agentloop/integrations/omnigent/telemetry.py +422 -0
- agentloop/integrations/openai.py +415 -0
- agentloop/integrations/openai_agents.py +291 -0
- agentloop/integrations/sdk.py +82 -0
- agentloop/integrations/vercel_ai.py +83 -0
- agentloop/interoperability/__init__.py +5 -0
- agentloop/interoperability/artifacts.py +112 -0
- agentloop/interoperability/cli.py +59 -0
- agentloop/interoperability/cohort_evidence.py +175 -0
- agentloop/interoperability/cohorts.py +681 -0
- agentloop/interoperability/contracts.py +427 -0
- agentloop/interoperability/coordination.py +554 -0
- agentloop/interoperability/evidence.py +215 -0
- agentloop/interoperability/otlp_jsonl.py +775 -0
- agentloop/interoperability/otlp_validation.py +496 -0
- agentloop/interoperability/registry.py +561 -0
- agentloop/interoperability/validation.py +333 -0
- agentloop/intervention_service.py +110 -0
- agentloop/intervention_store.py +118 -0
- agentloop/interventions.py +190 -0
- agentloop/issues.py +90 -0
- agentloop/judgment_adapters.py +125 -0
- agentloop/judgment_benchmark_types.py +98 -0
- agentloop/judgment_benchmarks.py +606 -0
- agentloop/judgment_types.py +241 -0
- agentloop/judgment_views.py +38 -0
- agentloop/judgments.py +342 -0
- agentloop/loop_guards.py +274 -0
- agentloop/loop_types.py +66 -0
- agentloop/markdown.py +102 -0
- agentloop/metrics.py +342 -0
- agentloop/migrations.py +303 -0
- agentloop/model_routing.py +470 -0
- agentloop/onboarding.py +315 -0
- agentloop/operations.py +44 -0
- agentloop/optimizer.py +199 -0
- agentloop/otel.py +715 -0
- agentloop/otel_ids.py +81 -0
- agentloop/otel_semantics.py +134 -0
- agentloop/parallelism.py +141 -0
- agentloop/patches.py +603 -0
- agentloop/plan_export.py +126 -0
- agentloop/policy_promotion.py +325 -0
- agentloop/promotion_evidence.py +304 -0
- agentloop/promotion_types.py +209 -0
- agentloop/quality.py +446 -0
- agentloop/ranking.py +529 -0
- agentloop/recovery_evidence.py +96 -0
- agentloop/replay.py +753 -0
- agentloop/retention.py +375 -0
- agentloop/retention_types.py +113 -0
- agentloop/reuse_experiments.py +247 -0
- agentloop/routing_checks.py +116 -0
- agentloop/routing_types.py +241 -0
- agentloop/rules.py +396 -0
- agentloop/runtime.py +286 -0
- agentloop/savings.py +158 -0
- agentloop/scheduling_plan.py +99 -0
- agentloop/scheduling_types.py +169 -0
- agentloop/schema.py +257 -0
- agentloop/semantic_rules.py +200 -0
- agentloop/semantic_waste.py +345 -0
- agentloop/semantic_waste_types.py +88 -0
- agentloop/server.py +494 -0
- agentloop/store.py +1429 -0
- agentloop/structural_experiments.py +431 -0
- agentloop/structural_types.py +113 -0
- agentloop/structured_quality.py +1020 -0
- agentloop/studies.py +629 -0
- agentloop/study_statistics.py +58 -0
- agentloop/substitution_evidence.py +199 -0
- agentloop/substitution_export.py +190 -0
- agentloop/substitution_reports.py +407 -0
- agentloop/substitution_types.py +284 -0
- agentloop/substitutions.py +267 -0
- agentloop/timing.py +142 -0
- agentloop/tokens.py +188 -0
- agentloop/tool_scheduling.py +410 -0
- agentloop/trace_sources.py +31 -0
- agentloop/tracer.py +494 -0
- agentloop/value.py +306 -0
- agentloop/version.py +3 -0
- agentloop/workflow_types.py +264 -0
- agentloop/workflows.py +284 -0
- agentloop_profiler-0.1.0.dist-info/METADATA +513 -0
- agentloop_profiler-0.1.0.dist-info/RECORD +166 -0
- agentloop_profiler-0.1.0.dist-info/WHEEL +5 -0
- agentloop_profiler-0.1.0.dist-info/entry_points.txt +2 -0
- agentloop_profiler-0.1.0.dist-info/licenses/LICENSE +201 -0
- agentloop_profiler-0.1.0.dist-info/licenses/THIRD_PARTY_LICENSES.md +101 -0
- agentloop_profiler-0.1.0.dist-info/top_level.txt +2 -0
- dashboard/__init__.py +5 -0
- dashboard/app.py +922 -0
- dashboard/streamlit_compat.py +63 -0
- dashboard/trace_upload.py +57 -0
- dashboard/value_view.py +105 -0
|
@@ -0,0 +1,340 @@
|
|
|
1
|
+
"""Bounded cohort windows built from native traces or existing aggregate artifacts."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import copy
|
|
6
|
+
import hashlib
|
|
7
|
+
import json
|
|
8
|
+
from collections import Counter
|
|
9
|
+
from dataclasses import asdict
|
|
10
|
+
|
|
11
|
+
from agentloop.aggregate_stats import AggregateConfig, number, total
|
|
12
|
+
from agentloop.aggregates import TraceAggregate
|
|
13
|
+
from agentloop.drift_types import QualityMetric, WindowSpec, label, timestamp
|
|
14
|
+
from agentloop.onboarding import validate_capture
|
|
15
|
+
from agentloop.operations import operation_kind
|
|
16
|
+
from agentloop.retention import read_retention
|
|
17
|
+
from agentloop.rules import BUILTIN_RULES
|
|
18
|
+
from agentloop.workflow_types import stage_summary, workflow_summary
|
|
19
|
+
|
|
20
|
+
DRIFT_VERSION = "1.0"
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def digest(value):
|
|
24
|
+
return hashlib.sha256(
|
|
25
|
+
json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False).encode()
|
|
26
|
+
).hexdigest()
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def seal(value):
|
|
30
|
+
result = copy.deepcopy(value)
|
|
31
|
+
result["sha256"] = digest(result)
|
|
32
|
+
return result
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def verify(value, kind):
|
|
36
|
+
if (
|
|
37
|
+
not isinstance(value, dict)
|
|
38
|
+
or value.get("schema_version") != DRIFT_VERSION
|
|
39
|
+
or value.get("kind") != kind
|
|
40
|
+
):
|
|
41
|
+
raise ValueError(f"unsupported {kind} artifact")
|
|
42
|
+
content = dict(value)
|
|
43
|
+
if content.pop("sha256", None) != digest(content):
|
|
44
|
+
raise ValueError(f"{kind} content hash mismatch")
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _mix():
|
|
48
|
+
return {"counts": Counter(), "overflow": 0, "missing": 0, "total": 0}
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _mix_add(mix, value):
|
|
52
|
+
mix["total"] += 1
|
|
53
|
+
if value is None:
|
|
54
|
+
mix["missing"] += 1
|
|
55
|
+
return
|
|
56
|
+
key = digest(value)
|
|
57
|
+
if key in mix["counts"] or len(mix["counts"]) < 128:
|
|
58
|
+
mix["counts"][key] += 1
|
|
59
|
+
else:
|
|
60
|
+
mix["overflow"] += 1
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
class WindowBuilder:
|
|
64
|
+
def __init__(
|
|
65
|
+
self,
|
|
66
|
+
spec: WindowSpec,
|
|
67
|
+
*,
|
|
68
|
+
quality_metrics: tuple[QualityMetric, ...] = (),
|
|
69
|
+
config: AggregateConfig | None = None,
|
|
70
|
+
include_findings=False,
|
|
71
|
+
):
|
|
72
|
+
if not isinstance(spec, WindowSpec):
|
|
73
|
+
raise TypeError("spec must be WindowSpec")
|
|
74
|
+
if (
|
|
75
|
+
not isinstance(quality_metrics, tuple)
|
|
76
|
+
or len(quality_metrics) > 16
|
|
77
|
+
or any(not isinstance(metric, QualityMetric) for metric in quality_metrics)
|
|
78
|
+
):
|
|
79
|
+
raise ValueError("quality_metrics must be a tuple of at most 16 declarations")
|
|
80
|
+
if len({metric.name for metric in quality_metrics}) != len(quality_metrics):
|
|
81
|
+
raise ValueError("quality metric names must be unique")
|
|
82
|
+
if type(include_findings) is not bool:
|
|
83
|
+
raise ValueError("include_findings must be boolean")
|
|
84
|
+
self.spec = spec
|
|
85
|
+
self.quality_metrics = tuple(sorted(quality_metrics, key=lambda metric: metric.name))
|
|
86
|
+
self.config = AggregateConfig() if config is None else config
|
|
87
|
+
if not isinstance(self.config, AggregateConfig):
|
|
88
|
+
raise TypeError("config must be AggregateConfig")
|
|
89
|
+
self.include_findings = include_findings
|
|
90
|
+
self._cohorts = {}
|
|
91
|
+
self._chain = digest([])
|
|
92
|
+
self._sources = 0
|
|
93
|
+
|
|
94
|
+
def _cohort(self, cohort):
|
|
95
|
+
label(cohort, "cohort")
|
|
96
|
+
if cohort not in self._cohorts and len(self._cohorts) >= 32:
|
|
97
|
+
raise ValueError("a drift window supports at most 32 explicit cohorts")
|
|
98
|
+
if cohort in self._cohorts:
|
|
99
|
+
return copy.deepcopy(self._cohorts[cohort])
|
|
100
|
+
return {
|
|
101
|
+
"aggregate": TraceAggregate(self.config),
|
|
102
|
+
"flags": {
|
|
103
|
+
key: {"known": 0, "true": 0}
|
|
104
|
+
for key in ("timeout", "abstention", "finding", "instrumentation")
|
|
105
|
+
},
|
|
106
|
+
"quality": {metric.name: {"count": 0, "sum": 0.0} for metric in self.quality_metrics},
|
|
107
|
+
"mixes": {key: _mix() for key in ("model", "stage", "outcome", "decision_outcome")},
|
|
108
|
+
"individual_evidence_missing_runs": 0,
|
|
109
|
+
"producer_markers": {"synthetic": 0, "declared_real": 0, "unspecified": 0},
|
|
110
|
+
"time_membership": {
|
|
111
|
+
"native_ended_at_verified_runs": 0,
|
|
112
|
+
"host_assigned_aggregate_runs": 0,
|
|
113
|
+
},
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
def add(self, trace, *, cohort="all", timeout=None, abstention=None, quality=None):
|
|
117
|
+
ended = timestamp(trace.ended_at)
|
|
118
|
+
if not timestamp(self.spec.started_at) <= ended < timestamp(self.spec.ended_at):
|
|
119
|
+
raise ValueError("trace completion is outside the declared window")
|
|
120
|
+
for value in (timeout, abstention):
|
|
121
|
+
if value is not None and type(value) is not bool:
|
|
122
|
+
raise ValueError("timeout/abstention must be boolean or None")
|
|
123
|
+
quality = {} if quality is None else quality
|
|
124
|
+
if not isinstance(quality, dict) or not set(quality) <= {
|
|
125
|
+
metric.name for metric in self.quality_metrics
|
|
126
|
+
}:
|
|
127
|
+
raise ValueError("quality values must match declared metrics")
|
|
128
|
+
for value in quality.values():
|
|
129
|
+
if value is not None and (
|
|
130
|
+
type(value) not in {int, float} or not 0 <= number(value) <= 1
|
|
131
|
+
):
|
|
132
|
+
raise ValueError("quality values must be in [0,1] or None")
|
|
133
|
+
owned = self._cohort(cohort)
|
|
134
|
+
owned["aggregate"].add(trace)
|
|
135
|
+
owned["time_membership"]["native_ended_at_verified_runs"] += 1
|
|
136
|
+
synthetic = trace.metadata.get("synthetic")
|
|
137
|
+
owned["producer_markers"][
|
|
138
|
+
"synthetic"
|
|
139
|
+
if synthetic is True
|
|
140
|
+
else "declared_real"
|
|
141
|
+
if synthetic is False
|
|
142
|
+
else "unspecified"
|
|
143
|
+
] += 1
|
|
144
|
+
retained = read_retention(trace)
|
|
145
|
+
complete = retained is None or retained["complete_evidence"]
|
|
146
|
+
for key, value in (("timeout", timeout), ("abstention", abstention)):
|
|
147
|
+
if value is not None:
|
|
148
|
+
owned["flags"][key]["known"] += 1
|
|
149
|
+
owned["flags"][key]["true"] += value
|
|
150
|
+
for key, value in quality.items():
|
|
151
|
+
if value is not None:
|
|
152
|
+
owned["quality"][key]["count"] += 1
|
|
153
|
+
owned["quality"][key]["sum"] = total(owned["quality"][key]["sum"], value)
|
|
154
|
+
owned["flags"]["instrumentation"]["known"] += 1
|
|
155
|
+
if complete:
|
|
156
|
+
validation = validate_capture(trace)
|
|
157
|
+
missing_codes = {
|
|
158
|
+
"no_execution_spans",
|
|
159
|
+
"unsupported_operation",
|
|
160
|
+
"usage_not_exact",
|
|
161
|
+
"model_identity_missing",
|
|
162
|
+
"trace_boundary_missing",
|
|
163
|
+
"span_timing_unavailable",
|
|
164
|
+
"span_outside_boundary",
|
|
165
|
+
"retained_evidence_incomplete",
|
|
166
|
+
}
|
|
167
|
+
owned["flags"]["instrumentation"]["true"] += not any(
|
|
168
|
+
check["severity"] == "error" or check["code"] in missing_codes
|
|
169
|
+
for check in validation["checks"]
|
|
170
|
+
)
|
|
171
|
+
for event in trace.events:
|
|
172
|
+
if operation_kind(event) == "model":
|
|
173
|
+
_mix_add(owned["mixes"]["model"], event.model)
|
|
174
|
+
stage = stage_summary(event)
|
|
175
|
+
_mix_add(
|
|
176
|
+
owned["mixes"]["stage"],
|
|
177
|
+
[stage.get("stage_id"), stage.get("version")]
|
|
178
|
+
if stage and stage.get("stage_id")
|
|
179
|
+
else None,
|
|
180
|
+
)
|
|
181
|
+
if operation_kind(event) in {"classifier", "rule"}:
|
|
182
|
+
_mix_add(
|
|
183
|
+
owned["mixes"]["decision_outcome"], stage.get("outcome") if stage else None
|
|
184
|
+
)
|
|
185
|
+
execution = workflow_summary(trace.metadata)
|
|
186
|
+
_mix_add(owned["mixes"]["outcome"], execution.get("outcome") if execution else None)
|
|
187
|
+
if self.include_findings and len(trace.events) <= 2000:
|
|
188
|
+
report = trace.report()
|
|
189
|
+
if report.get("analysis_complete"):
|
|
190
|
+
owned["flags"]["finding"]["known"] += 1
|
|
191
|
+
owned["flags"]["finding"]["true"] += bool(report["finding_candidates"])
|
|
192
|
+
else:
|
|
193
|
+
owned["individual_evidence_missing_runs"] += 1
|
|
194
|
+
source_hash = digest(trace.to_dict())
|
|
195
|
+
chain = digest([self._chain, source_hash, cohort, timeout, abstention, quality])
|
|
196
|
+
# All validation/analysis/hash work precedes mutation of builder state.
|
|
197
|
+
self._cohorts[cohort] = owned
|
|
198
|
+
self._chain, self._sources = chain, self._sources + 1
|
|
199
|
+
|
|
200
|
+
def add_aggregate(self, aggregate: TraceAggregate, *, cohort="all", time_assignment_ref):
|
|
201
|
+
"""Existing aggregates contribute original metrics, with missing per-run extras explicit."""
|
|
202
|
+
if not isinstance(aggregate, TraceAggregate):
|
|
203
|
+
raise TypeError("aggregate must be TraceAggregate")
|
|
204
|
+
label(time_assignment_ref, "time_assignment_ref")
|
|
205
|
+
owned = self._cohort(cohort)
|
|
206
|
+
owned["aggregate"].merge(aggregate)
|
|
207
|
+
report = aggregate.to_dict()
|
|
208
|
+
owned["individual_evidence_missing_runs"] += report["counts"]["runs"]
|
|
209
|
+
owned["time_membership"]["host_assigned_aggregate_runs"] += report["counts"]["runs"]
|
|
210
|
+
owned["producer_markers"]["unspecified"] += report["counts"]["runs"]
|
|
211
|
+
chain = digest([self._chain, report["sha256"], cohort, time_assignment_ref])
|
|
212
|
+
self._cohorts[cohort] = owned
|
|
213
|
+
self._chain, self._sources = chain, self._sources + 1
|
|
214
|
+
|
|
215
|
+
def to_dict(self):
|
|
216
|
+
cohorts = copy.deepcopy(self._cohorts)
|
|
217
|
+
for values in cohorts.values():
|
|
218
|
+
values["aggregate"] = values["aggregate"].to_dict()
|
|
219
|
+
for mix in values["mixes"].values():
|
|
220
|
+
mix["counts"] = dict(mix["counts"])
|
|
221
|
+
return seal(
|
|
222
|
+
{
|
|
223
|
+
"schema_version": DRIFT_VERSION,
|
|
224
|
+
"kind": "agentloop.drift_window",
|
|
225
|
+
"spec": asdict(self.spec),
|
|
226
|
+
"quality_metrics": [asdict(metric) for metric in self.quality_metrics],
|
|
227
|
+
"settings": {
|
|
228
|
+
"include_findings": self.include_findings,
|
|
229
|
+
"rule_versions": [[rule.rule_id, rule.version] for rule in BUILTIN_RULES]
|
|
230
|
+
if self.include_findings
|
|
231
|
+
else None,
|
|
232
|
+
},
|
|
233
|
+
"cohorts": cohorts,
|
|
234
|
+
"source_count": self._sources,
|
|
235
|
+
"source_chain_sha256": self._chain,
|
|
236
|
+
"scope": "Supplied disjoint records only; cohort labels and quality/timeout/abstention annotations are host observations.",
|
|
237
|
+
}
|
|
238
|
+
)
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
def validate_window(value):
|
|
242
|
+
verify(value, "agentloop.drift_window")
|
|
243
|
+
WindowSpec(**value["spec"])
|
|
244
|
+
metrics = tuple(QualityMetric(**item) for item in value["quality_metrics"])
|
|
245
|
+
if (
|
|
246
|
+
type(value["source_count"]) is not int
|
|
247
|
+
or value["source_count"] < 0
|
|
248
|
+
or not isinstance(value["source_chain_sha256"], str)
|
|
249
|
+
or len(value["source_chain_sha256"]) != 64
|
|
250
|
+
or any(char not in "0123456789abcdef" for char in value["source_chain_sha256"])
|
|
251
|
+
):
|
|
252
|
+
raise ValueError("invalid source chain")
|
|
253
|
+
settings = value["settings"]
|
|
254
|
+
if (
|
|
255
|
+
not isinstance(settings, dict)
|
|
256
|
+
or set(settings) != {"include_findings", "rule_versions"}
|
|
257
|
+
or type(settings["include_findings"]) is not bool
|
|
258
|
+
):
|
|
259
|
+
raise ValueError("invalid window analysis settings")
|
|
260
|
+
if settings["include_findings"]:
|
|
261
|
+
if not isinstance(settings["rule_versions"], list) or len(settings["rule_versions"]) > 128:
|
|
262
|
+
raise ValueError("invalid finding rule versions")
|
|
263
|
+
for entry in settings["rule_versions"]:
|
|
264
|
+
if not isinstance(entry, list) or len(entry) != 2:
|
|
265
|
+
raise ValueError("invalid finding rule version entry")
|
|
266
|
+
for field in entry:
|
|
267
|
+
label(field, "finding rule version")
|
|
268
|
+
elif settings["rule_versions"] is not None:
|
|
269
|
+
raise ValueError("unexpected finding rule versions")
|
|
270
|
+
if (
|
|
271
|
+
len(metrics) > 16
|
|
272
|
+
or len({metric.name for metric in metrics}) != len(metrics)
|
|
273
|
+
or not isinstance(value["cohorts"], dict)
|
|
274
|
+
or len(value["cohorts"]) > 32
|
|
275
|
+
):
|
|
276
|
+
raise ValueError("invalid drift window dimensions")
|
|
277
|
+
for cohort, data in value["cohorts"].items():
|
|
278
|
+
label(cohort, "cohort")
|
|
279
|
+
report = TraceAggregate.from_dict(data["aggregate"]).to_dict()
|
|
280
|
+
runs = report["counts"]["runs"]
|
|
281
|
+
markers = data["producer_markers"]
|
|
282
|
+
if (
|
|
283
|
+
set(markers) != {"synthetic", "declared_real", "unspecified"}
|
|
284
|
+
or any(type(count) is not int or count < 0 for count in markers.values())
|
|
285
|
+
or sum(markers.values()) != runs
|
|
286
|
+
):
|
|
287
|
+
raise ValueError("invalid producer marker counts")
|
|
288
|
+
membership = data["time_membership"]
|
|
289
|
+
if (
|
|
290
|
+
set(membership) != {"native_ended_at_verified_runs", "host_assigned_aggregate_runs"}
|
|
291
|
+
or any(type(count) is not int or count < 0 for count in membership.values())
|
|
292
|
+
or sum(membership.values()) != runs
|
|
293
|
+
):
|
|
294
|
+
raise ValueError("invalid time membership counts")
|
|
295
|
+
if (
|
|
296
|
+
set(data["quality"]) != {metric.name for metric in metrics}
|
|
297
|
+
or set(data["flags"]) != {"timeout", "abstention", "finding", "instrumentation"}
|
|
298
|
+
or set(data["mixes"]) != {"model", "stage", "outcome", "decision_outcome"}
|
|
299
|
+
):
|
|
300
|
+
raise ValueError("invalid window metric fields")
|
|
301
|
+
for flag in data["flags"].values():
|
|
302
|
+
if (
|
|
303
|
+
any(type(flag[key]) is not int for key in ("known", "true"))
|
|
304
|
+
or not 0 <= flag["true"] <= flag["known"] <= runs
|
|
305
|
+
):
|
|
306
|
+
raise ValueError("invalid window flag denominator")
|
|
307
|
+
for quality in data["quality"].values():
|
|
308
|
+
if (
|
|
309
|
+
type(quality["count"]) is not int
|
|
310
|
+
or not 0 <= quality["count"] <= runs
|
|
311
|
+
or not 0 <= number(quality["sum"]) <= quality["count"]
|
|
312
|
+
):
|
|
313
|
+
raise ValueError("invalid window quality denominator")
|
|
314
|
+
for mix in data["mixes"].values():
|
|
315
|
+
counts = mix["counts"]
|
|
316
|
+
if (
|
|
317
|
+
not isinstance(counts, dict)
|
|
318
|
+
or len(counts) > 128
|
|
319
|
+
or any(
|
|
320
|
+
not isinstance(key, str)
|
|
321
|
+
or len(key) != 64
|
|
322
|
+
or any(char not in "0123456789abcdef" for char in key)
|
|
323
|
+
for key in counts
|
|
324
|
+
)
|
|
325
|
+
):
|
|
326
|
+
raise ValueError("invalid window mix keys")
|
|
327
|
+
if (
|
|
328
|
+
any(
|
|
329
|
+
type(item) is not int or item < 0
|
|
330
|
+
for item in [*counts.values(), mix["missing"], mix["overflow"], mix["total"]]
|
|
331
|
+
)
|
|
332
|
+
or sum(counts.values()) + mix["missing"] + mix["overflow"] != mix["total"]
|
|
333
|
+
):
|
|
334
|
+
raise ValueError("invalid window mix denominator")
|
|
335
|
+
if (
|
|
336
|
+
type(data["individual_evidence_missing_runs"]) is not int
|
|
337
|
+
or not 0 <= data["individual_evidence_missing_runs"] <= runs
|
|
338
|
+
):
|
|
339
|
+
raise ValueError("invalid individual evidence count")
|
|
340
|
+
return copy.deepcopy(value)
|
agentloop/entrypoint.py
ADDED
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
import typer
|
|
6
|
+
from rich.table import Table
|
|
7
|
+
|
|
8
|
+
from agentloop.cli import _build_quality_report_from_file, _load_trace, _write_json, app, console
|
|
9
|
+
from agentloop.events import AgentEvent
|
|
10
|
+
from agentloop.findings import build_diagnosis
|
|
11
|
+
from agentloop.html_report import analysis_to_html
|
|
12
|
+
from agentloop.optimizer import build_optimization_plan
|
|
13
|
+
from agentloop.ranking import RankingSort
|
|
14
|
+
from agentloop.replay import ReplayGates, build_replay_report
|
|
15
|
+
from agentloop.timing import format_duration_ms
|
|
16
|
+
from agentloop.tracer import AgentTrace
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _quickstart_trace() -> AgentTrace:
|
|
20
|
+
"""Build a deterministic synthetic trace with several obvious bottlenecks."""
|
|
21
|
+
run_id = "run_agentloop_quickstart"
|
|
22
|
+
trace = AgentTrace(
|
|
23
|
+
name="agentloop_quickstart",
|
|
24
|
+
run_id=run_id,
|
|
25
|
+
metadata={"synthetic": True, "source": "agentloop_quickstart"},
|
|
26
|
+
started_at="2026-01-01T00:00:00+00:00",
|
|
27
|
+
ended_at="2026-01-01T00:00:01+00:00",
|
|
28
|
+
elapsed_ms=1000.0,
|
|
29
|
+
)
|
|
30
|
+
events = [
|
|
31
|
+
("tool_call", "retrieve", 0.0, 180.0, None, 0, 0),
|
|
32
|
+
("tool_call", "retrieve", 180.0, 360.0, None, 0, 0),
|
|
33
|
+
("tool_call", "retrieve", 360.0, 540.0, None, 0, 0),
|
|
34
|
+
("model_call", "summarize", 540.0, 650.0, "gpt-4.1", 700, 80),
|
|
35
|
+
("model_call", "summarize", 650.0, 760.0, "gpt-4.1", 700, 80),
|
|
36
|
+
("model_call", "summarize", 760.0, 870.0, "gpt-4.1", 700, 80),
|
|
37
|
+
("retry", "repair_json", 870.0, 930.0, None, 0, 0),
|
|
38
|
+
]
|
|
39
|
+
for index, (
|
|
40
|
+
event_type,
|
|
41
|
+
name,
|
|
42
|
+
start_ms,
|
|
43
|
+
end_ms,
|
|
44
|
+
model,
|
|
45
|
+
input_tokens,
|
|
46
|
+
output_tokens,
|
|
47
|
+
) in enumerate(events, start=1):
|
|
48
|
+
start_seconds = start_ms / 1000
|
|
49
|
+
end_seconds = end_ms / 1000
|
|
50
|
+
trace.add_event(
|
|
51
|
+
AgentEvent(
|
|
52
|
+
event_id=f"evt_quickstart_{index:02d}",
|
|
53
|
+
run_id=run_id,
|
|
54
|
+
event_type=event_type,
|
|
55
|
+
name=name,
|
|
56
|
+
started_at=f"2026-01-01T00:00:{start_seconds:06.3f}+00:00",
|
|
57
|
+
ended_at=f"2026-01-01T00:00:{end_seconds:06.3f}+00:00",
|
|
58
|
+
duration_ms=end_ms - start_ms,
|
|
59
|
+
model=model,
|
|
60
|
+
input_tokens=input_tokens,
|
|
61
|
+
output_tokens=output_tokens,
|
|
62
|
+
input_text=("stable research instruction " * 30 if model else None),
|
|
63
|
+
)
|
|
64
|
+
)
|
|
65
|
+
return trace
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _print_analysis(trace: AgentTrace, diagnosis: dict, plan: dict) -> None:
|
|
69
|
+
report = trace.report()
|
|
70
|
+
summary = diagnosis["summary"]
|
|
71
|
+
table = Table(title=f"AgentLoop Analysis: {trace.name}")
|
|
72
|
+
table.add_column("Metric")
|
|
73
|
+
table.add_column("Value")
|
|
74
|
+
table.add_row("Runtime", format_duration_ms(report["total_runtime_ms"], precision=3))
|
|
75
|
+
if report.get("external_evidence"):
|
|
76
|
+
source = report["external_evidence"]["source"]
|
|
77
|
+
table.add_row("External source", str(source.get("system", "unknown")))
|
|
78
|
+
table.add_row("Execution status", report["external_evidence"]["execution_status"])
|
|
79
|
+
table.add_row("Timing coverage", report["timing_status"])
|
|
80
|
+
table.add_row("Findings", str(summary["finding_count"]))
|
|
81
|
+
table.add_row("High severity", str(summary["high_severity_count"]))
|
|
82
|
+
table.add_row("Patchable", str(summary["patchable_count"]))
|
|
83
|
+
table.add_row("Retries", str(report["retry_count"]))
|
|
84
|
+
table.add_row("Input tokens", str(report["input_tokens"]))
|
|
85
|
+
console.print(table)
|
|
86
|
+
|
|
87
|
+
if report.get("rule_errors"):
|
|
88
|
+
console.print(
|
|
89
|
+
"Analysis incomplete; failed rules: "
|
|
90
|
+
+ ", ".join(error["rule_id"] for error in report["rule_errors"]),
|
|
91
|
+
markup=False,
|
|
92
|
+
)
|
|
93
|
+
|
|
94
|
+
findings = diagnosis.get("findings", [])
|
|
95
|
+
if not findings:
|
|
96
|
+
console.print("No optimization findings detected.")
|
|
97
|
+
return
|
|
98
|
+
console.print("\nTop findings:")
|
|
99
|
+
for finding in findings[:5]:
|
|
100
|
+
console.print(
|
|
101
|
+
f"- [{finding['ranking']['status'].replace('_', ' ')}] {finding['title']} ({finding['confidence']} confidence)",
|
|
102
|
+
markup=False,
|
|
103
|
+
)
|
|
104
|
+
aggregation = plan.get("savings_aggregation", {})
|
|
105
|
+
if aggregation.get("selection_optimal") is False:
|
|
106
|
+
console.print("Savings selection includes an approximate large-component fallback.")
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def _analysis_payload(trace: AgentTrace, *, sort_by="priority") -> dict:
|
|
110
|
+
report = trace.report()
|
|
111
|
+
diagnosis = build_diagnosis(trace, sort_by=sort_by)
|
|
112
|
+
plan = build_optimization_plan(trace, report=report)
|
|
113
|
+
return {
|
|
114
|
+
"trace": trace.to_dict(),
|
|
115
|
+
"report": report,
|
|
116
|
+
"diagnosis": diagnosis,
|
|
117
|
+
"optimization": plan,
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
@app.command("quickstart")
|
|
122
|
+
def quickstart_command(
|
|
123
|
+
out: Path = typer.Option(
|
|
124
|
+
Path("runs/agentloop_quickstart.json"),
|
|
125
|
+
help="Where to write the deterministic synthetic trace.",
|
|
126
|
+
),
|
|
127
|
+
json_out: Path | None = typer.Option(
|
|
128
|
+
None, help="Optional path for the combined analysis JSON."
|
|
129
|
+
),
|
|
130
|
+
) -> None:
|
|
131
|
+
"""Generate one offline trace and show useful findings with no account or API key."""
|
|
132
|
+
trace = _quickstart_trace()
|
|
133
|
+
trace.export_json(out)
|
|
134
|
+
payload = _analysis_payload(trace)
|
|
135
|
+
_print_analysis(trace, payload["diagnosis"], payload["optimization"])
|
|
136
|
+
console.print(f"\nSynthetic trace: {out}")
|
|
137
|
+
console.print(f"Next: agentloop analyze {out}")
|
|
138
|
+
if json_out is not None:
|
|
139
|
+
_write_json(json_out, payload)
|
|
140
|
+
console.print(f"Analysis JSON: {json_out}")
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
@app.command("analyze")
|
|
144
|
+
def analyze_command(
|
|
145
|
+
path: Path,
|
|
146
|
+
json_out: Path | None = typer.Option(
|
|
147
|
+
None, help="Optional path for report, diagnosis, and optimization JSON."
|
|
148
|
+
),
|
|
149
|
+
html_out: Path | None = typer.Option(
|
|
150
|
+
None, "--html", help="Write a single offline HTML report."
|
|
151
|
+
),
|
|
152
|
+
baseline: Path | None = typer.Option(
|
|
153
|
+
None, help="Optional baseline trace; the positional trace is the candidate."
|
|
154
|
+
),
|
|
155
|
+
quality_fixtures: Path | None = typer.Option(
|
|
156
|
+
None, help="Quality fixtures for the optional baseline comparison."
|
|
157
|
+
),
|
|
158
|
+
min_quality_score: float | None = typer.Option(None, min=0, max=1),
|
|
159
|
+
include_content: bool = typer.Option(
|
|
160
|
+
False, help="Include raw event text and full event metadata in HTML."
|
|
161
|
+
),
|
|
162
|
+
sort_by: RankingSort = typer.Option(
|
|
163
|
+
RankingSort.PRIORITY, help="Order findings by an explicit ranking dimension."
|
|
164
|
+
),
|
|
165
|
+
) -> None:
|
|
166
|
+
"""Analyze one existing AgentLoop trace in a single command."""
|
|
167
|
+
trace = _load_trace(path, param_hint="path")
|
|
168
|
+
if (quality_fixtures is not None or min_quality_score is not None) and baseline is None:
|
|
169
|
+
raise typer.BadParameter("quality comparison options require --baseline")
|
|
170
|
+
if include_content and html_out is None:
|
|
171
|
+
raise typer.BadParameter("--include-content requires --html")
|
|
172
|
+
payload = _analysis_payload(trace, sort_by=sort_by.value)
|
|
173
|
+
if baseline is not None:
|
|
174
|
+
baseline_trace = _load_trace(baseline, param_hint="--baseline")
|
|
175
|
+
quality = (
|
|
176
|
+
_build_quality_report_from_file(
|
|
177
|
+
quality_fixtures,
|
|
178
|
+
param_hint="--quality-fixtures",
|
|
179
|
+
baseline_trace=baseline_trace,
|
|
180
|
+
candidate_trace=trace,
|
|
181
|
+
min_score=min_quality_score,
|
|
182
|
+
)
|
|
183
|
+
if quality_fixtures is not None
|
|
184
|
+
else None
|
|
185
|
+
)
|
|
186
|
+
payload["replay"] = build_replay_report(
|
|
187
|
+
baseline_trace,
|
|
188
|
+
trace,
|
|
189
|
+
gates=ReplayGates(min_quality_score=min_quality_score),
|
|
190
|
+
quality_report=quality,
|
|
191
|
+
)
|
|
192
|
+
_print_analysis(trace, payload["diagnosis"], payload["optimization"])
|
|
193
|
+
if json_out is not None:
|
|
194
|
+
_write_json(json_out, payload)
|
|
195
|
+
console.print(f"Wrote analysis JSON to {json_out}")
|
|
196
|
+
if html_out is not None:
|
|
197
|
+
html_out.parent.mkdir(parents=True, exist_ok=True)
|
|
198
|
+
html_out.write_text(
|
|
199
|
+
analysis_to_html(payload, include_content=include_content), encoding="utf-8"
|
|
200
|
+
)
|
|
201
|
+
console.print(f"Wrote HTML report to {html_out}")
|
agentloop/estimates.py
ADDED
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
"""Versioned, uncalibrated savings models and self-contained evidence snapshots."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
from dataclasses import dataclass
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
from agentloop.costs import is_cost_evaluable
|
|
10
|
+
from agentloop.markdown import markdown_code_span, markdown_text
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@dataclass(frozen=True)
|
|
14
|
+
class Estimator:
|
|
15
|
+
formula: str
|
|
16
|
+
assumptions: tuple[str, ...]
|
|
17
|
+
parameters: tuple[tuple[str, float], ...] = ()
|
|
18
|
+
version: str = "1.0"
|
|
19
|
+
|
|
20
|
+
def parameter(self, name: str) -> float:
|
|
21
|
+
return dict(self.parameters)[name]
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
ESTIMATORS = {
|
|
25
|
+
"parallelize_tools": Estimator(
|
|
26
|
+
"sum_duration_ms - max_duration_ms",
|
|
27
|
+
("The formula applies assuming concurrent execution has negligible scheduling overhead.",),
|
|
28
|
+
),
|
|
29
|
+
"cache_context": Estimator(
|
|
30
|
+
"current_cost_usd * min(max_cost_fraction, repeated_context_ratio)",
|
|
31
|
+
(
|
|
32
|
+
"Repeated context can be reused without changing required outputs.",
|
|
33
|
+
"The reusable share of total priced cost approximates the repeated context ratio.",
|
|
34
|
+
"The provider or application supports the proposed cache.",
|
|
35
|
+
),
|
|
36
|
+
(("max_cost_fraction", 0.5),),
|
|
37
|
+
),
|
|
38
|
+
"add_schema_validation": Estimator(
|
|
39
|
+
"sum_duration_ms * latency_fraction",
|
|
40
|
+
(
|
|
41
|
+
"Most observed retries are preventable with validation or repair.",
|
|
42
|
+
"Added validation overhead is smaller than avoided retry time.",
|
|
43
|
+
),
|
|
44
|
+
(("latency_fraction", 0.8),),
|
|
45
|
+
),
|
|
46
|
+
"batch_model_calls": Estimator(
|
|
47
|
+
"sum_duration_ms * latency_fraction",
|
|
48
|
+
(
|
|
49
|
+
"Repeated calls accept a shared batch without changing item-level outputs.",
|
|
50
|
+
"Batch size, token limits, and provider throughput permit the estimated reduction.",
|
|
51
|
+
),
|
|
52
|
+
(("latency_fraction", 0.35),),
|
|
53
|
+
),
|
|
54
|
+
"route_to_smaller_model": Estimator(
|
|
55
|
+
"sum_duration_ms * latency_fraction",
|
|
56
|
+
(
|
|
57
|
+
"A smaller model meets the task's quality requirements.",
|
|
58
|
+
"Its serving latency is lower under comparable load.",
|
|
59
|
+
),
|
|
60
|
+
(("latency_fraction", 0.25),),
|
|
61
|
+
),
|
|
62
|
+
"split_large_step": Estimator(
|
|
63
|
+
"sum_duration_ms * latency_fraction",
|
|
64
|
+
(
|
|
65
|
+
"Context can be compressed or split without losing necessary information.",
|
|
66
|
+
"Extra orchestration and model calls do not consume the estimated savings.",
|
|
67
|
+
),
|
|
68
|
+
(("latency_fraction", 0.20),),
|
|
69
|
+
),
|
|
70
|
+
"runaway_loop": Estimator(
|
|
71
|
+
"sum_duration_ms * latency_fraction",
|
|
72
|
+
(
|
|
73
|
+
"Some repeated steps are unnecessary for successful task completion.",
|
|
74
|
+
"A guard can stop them without prematurely terminating useful work.",
|
|
75
|
+
),
|
|
76
|
+
(("latency_fraction", 0.30),),
|
|
77
|
+
),
|
|
78
|
+
"tool_oscillation": Estimator(
|
|
79
|
+
"sum_duration_ms * latency_fraction",
|
|
80
|
+
(
|
|
81
|
+
"Some alternating tool calls repeat equivalent state transitions.",
|
|
82
|
+
"A state-change check can remove them without losing required transitions.",
|
|
83
|
+
),
|
|
84
|
+
(("latency_fraction", 0.40),),
|
|
85
|
+
),
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def estimate_snapshot(candidate: Any, report: dict[str, Any], nodes: list[Any]) -> dict[str, Any]:
|
|
90
|
+
"""Copy the estimator and its inputs; never reconstruct historical records on read."""
|
|
91
|
+
spec = ESTIMATORS[candidate.rule_id]
|
|
92
|
+
affected = set(candidate.affected_nodes or [])
|
|
93
|
+
durations = [node.duration_ms for node in nodes if node.node_id in affected]
|
|
94
|
+
cost_status = report.get("cost_status", "complete")
|
|
95
|
+
return {
|
|
96
|
+
"estimator_id": candidate.rule_id,
|
|
97
|
+
"estimator_version": spec.version,
|
|
98
|
+
"method": "heuristic",
|
|
99
|
+
"confidence": candidate.confidence,
|
|
100
|
+
"calibrated": False,
|
|
101
|
+
"formula": spec.formula,
|
|
102
|
+
"parameters": dict(spec.parameters),
|
|
103
|
+
"inputs": {
|
|
104
|
+
"sum_duration_ms": sum(durations),
|
|
105
|
+
"max_duration_ms": max(durations, default=0.0),
|
|
106
|
+
"span_count": len(durations),
|
|
107
|
+
"current_cost_usd": (
|
|
108
|
+
report.get("estimated_cost_usd", 0.0) if is_cost_evaluable(cost_status) else None
|
|
109
|
+
),
|
|
110
|
+
"repeated_context_ratio": report.get("repeated_context_ratio", 0.0),
|
|
111
|
+
"cost_status": cost_status,
|
|
112
|
+
"token_status": report.get("token_status", "unspecified"),
|
|
113
|
+
},
|
|
114
|
+
"assumptions": list(dict.fromkeys([*(candidate.assumptions or []), *spec.assumptions])),
|
|
115
|
+
"unmodeled_metrics": (["latency"] if candidate.rule_id == "cache_context" else ["cost"]),
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def estimate_markdown(estimate: dict[str, Any] | None) -> list[str]:
|
|
120
|
+
if not estimate:
|
|
121
|
+
return []
|
|
122
|
+
calibration = "calibrated" if estimate.get("calibrated") is True else "uncalibrated"
|
|
123
|
+
return [
|
|
124
|
+
f"- Estimator: {markdown_code_span(estimate['estimator_id'])} "
|
|
125
|
+
f"version {markdown_text(estimate['estimator_version'])}",
|
|
126
|
+
f"- Method: {markdown_text(estimate['method'])} ({calibration}; predicted savings)",
|
|
127
|
+
f"- Savings formula: {markdown_code_span(estimate['formula'])}",
|
|
128
|
+
"- Parameters: "
|
|
129
|
+
+ markdown_code_span(json.dumps(estimate.get("parameters", {}), sort_keys=True)),
|
|
130
|
+
"- Assumptions: "
|
|
131
|
+
+ "; ".join(markdown_text(value) for value in estimate.get("assumptions", [])),
|
|
132
|
+
"",
|
|
133
|
+
]
|