agentloop-profiler 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agentloop/__init__.py +136 -0
- agentloop/ablation_protocol.py +229 -0
- agentloop/ablations.py +564 -0
- agentloop/aggregate_cli.py +88 -0
- agentloop/aggregate_stats.py +175 -0
- agentloop/aggregates.py +532 -0
- agentloop/audit.py +39 -0
- agentloop/autoinstrument.py +145 -0
- agentloop/batch_experiments.py +295 -0
- agentloop/batching.py +95 -0
- agentloop/budget_types.py +335 -0
- agentloop/budgets.py +539 -0
- agentloop/calibration.py +872 -0
- agentloop/checkpoint_state.py +306 -0
- agentloop/checkpoint_store.py +86 -0
- agentloop/ci.py +202 -0
- agentloop/cli.py +1314 -0
- agentloop/client.py +228 -0
- agentloop/completion.py +345 -0
- agentloop/completion_types.py +199 -0
- agentloop/config.py +50 -0
- agentloop/context_controls.py +443 -0
- agentloop/context_experiments.py +337 -0
- agentloop/context_types.py +216 -0
- agentloop/costs.py +667 -0
- agentloop/cursor_validation.py +48 -0
- agentloop/decorators.py +445 -0
- agentloop/demo.py +300 -0
- agentloop/doctor.py +341 -0
- agentloop/drift.py +361 -0
- agentloop/drift_cli.py +147 -0
- agentloop/drift_types.py +110 -0
- agentloop/drift_windows.py +340 -0
- agentloop/entrypoint.py +201 -0
- agentloop/estimates.py +133 -0
- agentloop/events.py +75 -0
- agentloop/experiment_artifacts.py +80 -0
- agentloop/experiment_observations.py +169 -0
- agentloop/experiment_reports.py +545 -0
- agentloop/experiment_types.py +355 -0
- agentloop/experiments.py +392 -0
- agentloop/exporters.py +117 -0
- agentloop/finding_benchmark_types.py +131 -0
- agentloop/finding_benchmarks.py +424 -0
- agentloop/findings.py +419 -0
- agentloop/graph.py +464 -0
- agentloop/harness.py +825 -0
- agentloop/harness_bench/__init__.py +1 -0
- agentloop/harness_bench/cli.py +77 -0
- agentloop/harness_bench/drivers.py +96 -0
- agentloop/harness_bench/offline.py +507 -0
- agentloop/harness_evidence.py +585 -0
- agentloop/harness_usage.py +91 -0
- agentloop/html_report.py +860 -0
- agentloop/integrations/__init__.py +1 -0
- agentloop/integrations/crewai.py +144 -0
- agentloop/integrations/harbor/__init__.py +1 -0
- agentloop/integrations/harbor/atif.py +982 -0
- agentloop/integrations/harbor/atif_validation.py +329 -0
- agentloop/integrations/harbor/cli.py +207 -0
- agentloop/integrations/harbor/manifest.py +86 -0
- agentloop/integrations/harbor/trial_evidence.py +196 -0
- agentloop/integrations/harbor/trials.py +1109 -0
- agentloop/integrations/harbor/verifier.py +118 -0
- agentloop/integrations/langgraph.py +150 -0
- agentloop/integrations/langgraph_harness.py +466 -0
- agentloop/integrations/langgraph_recovery.py +366 -0
- agentloop/integrations/omnigent/__init__.py +1 -0
- agentloop/integrations/omnigent/cli.py +78 -0
- agentloop/integrations/omnigent/evidence.py +93 -0
- agentloop/integrations/omnigent/telemetry.py +422 -0
- agentloop/integrations/openai.py +415 -0
- agentloop/integrations/openai_agents.py +291 -0
- agentloop/integrations/sdk.py +82 -0
- agentloop/integrations/vercel_ai.py +83 -0
- agentloop/interoperability/__init__.py +5 -0
- agentloop/interoperability/artifacts.py +112 -0
- agentloop/interoperability/cli.py +59 -0
- agentloop/interoperability/cohort_evidence.py +175 -0
- agentloop/interoperability/cohorts.py +681 -0
- agentloop/interoperability/contracts.py +427 -0
- agentloop/interoperability/coordination.py +554 -0
- agentloop/interoperability/evidence.py +215 -0
- agentloop/interoperability/otlp_jsonl.py +775 -0
- agentloop/interoperability/otlp_validation.py +496 -0
- agentloop/interoperability/registry.py +561 -0
- agentloop/interoperability/validation.py +333 -0
- agentloop/intervention_service.py +110 -0
- agentloop/intervention_store.py +118 -0
- agentloop/interventions.py +190 -0
- agentloop/issues.py +90 -0
- agentloop/judgment_adapters.py +125 -0
- agentloop/judgment_benchmark_types.py +98 -0
- agentloop/judgment_benchmarks.py +606 -0
- agentloop/judgment_types.py +241 -0
- agentloop/judgment_views.py +38 -0
- agentloop/judgments.py +342 -0
- agentloop/loop_guards.py +274 -0
- agentloop/loop_types.py +66 -0
- agentloop/markdown.py +102 -0
- agentloop/metrics.py +342 -0
- agentloop/migrations.py +303 -0
- agentloop/model_routing.py +470 -0
- agentloop/onboarding.py +315 -0
- agentloop/operations.py +44 -0
- agentloop/optimizer.py +199 -0
- agentloop/otel.py +715 -0
- agentloop/otel_ids.py +81 -0
- agentloop/otel_semantics.py +134 -0
- agentloop/parallelism.py +141 -0
- agentloop/patches.py +603 -0
- agentloop/plan_export.py +126 -0
- agentloop/policy_promotion.py +325 -0
- agentloop/promotion_evidence.py +304 -0
- agentloop/promotion_types.py +209 -0
- agentloop/quality.py +446 -0
- agentloop/ranking.py +529 -0
- agentloop/recovery_evidence.py +96 -0
- agentloop/replay.py +753 -0
- agentloop/retention.py +375 -0
- agentloop/retention_types.py +113 -0
- agentloop/reuse_experiments.py +247 -0
- agentloop/routing_checks.py +116 -0
- agentloop/routing_types.py +241 -0
- agentloop/rules.py +396 -0
- agentloop/runtime.py +286 -0
- agentloop/savings.py +158 -0
- agentloop/scheduling_plan.py +99 -0
- agentloop/scheduling_types.py +169 -0
- agentloop/schema.py +257 -0
- agentloop/semantic_rules.py +200 -0
- agentloop/semantic_waste.py +345 -0
- agentloop/semantic_waste_types.py +88 -0
- agentloop/server.py +494 -0
- agentloop/store.py +1429 -0
- agentloop/structural_experiments.py +431 -0
- agentloop/structural_types.py +113 -0
- agentloop/structured_quality.py +1020 -0
- agentloop/studies.py +629 -0
- agentloop/study_statistics.py +58 -0
- agentloop/substitution_evidence.py +199 -0
- agentloop/substitution_export.py +190 -0
- agentloop/substitution_reports.py +407 -0
- agentloop/substitution_types.py +284 -0
- agentloop/substitutions.py +267 -0
- agentloop/timing.py +142 -0
- agentloop/tokens.py +188 -0
- agentloop/tool_scheduling.py +410 -0
- agentloop/trace_sources.py +31 -0
- agentloop/tracer.py +494 -0
- agentloop/value.py +306 -0
- agentloop/version.py +3 -0
- agentloop/workflow_types.py +264 -0
- agentloop/workflows.py +284 -0
- agentloop_profiler-0.1.0.dist-info/METADATA +513 -0
- agentloop_profiler-0.1.0.dist-info/RECORD +166 -0
- agentloop_profiler-0.1.0.dist-info/WHEEL +5 -0
- agentloop_profiler-0.1.0.dist-info/entry_points.txt +2 -0
- agentloop_profiler-0.1.0.dist-info/licenses/LICENSE +201 -0
- agentloop_profiler-0.1.0.dist-info/licenses/THIRD_PARTY_LICENSES.md +101 -0
- agentloop_profiler-0.1.0.dist-info/top_level.txt +2 -0
- dashboard/__init__.py +5 -0
- dashboard/app.py +922 -0
- dashboard/streamlit_compat.py +63 -0
- dashboard/trace_upload.py +57 -0
- dashboard/value_view.py +105 -0
agentloop/rules.py
ADDED
|
@@ -0,0 +1,396 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
from collections.abc import Callable, Mapping
|
|
5
|
+
from copy import deepcopy
|
|
6
|
+
from dataclasses import dataclass, fields
|
|
7
|
+
from enum import Enum
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
from agentloop.batching import model_batch_groups
|
|
11
|
+
from agentloop.costs import is_cost_evaluable
|
|
12
|
+
from agentloop.estimates import ESTIMATORS, estimate_snapshot
|
|
13
|
+
from agentloop.graph import ExecutionGraph
|
|
14
|
+
from agentloop.parallelism import PARALLELISM_REWRITE
|
|
15
|
+
from agentloop.ranking import requirements
|
|
16
|
+
from agentloop.tokens import UNAVAILABLE
|
|
17
|
+
|
|
18
|
+
_LOGGER = logging.getLogger(__name__)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _safe_error_type(error: Exception) -> str:
|
|
22
|
+
"""Use public categories rather than exposing arbitrary implementation class names."""
|
|
23
|
+
for kind, label in (
|
|
24
|
+
(TypeError, "TypeError"),
|
|
25
|
+
(ValueError, "ValueError"),
|
|
26
|
+
(LookupError, "LookupError"),
|
|
27
|
+
(ArithmeticError, "ArithmeticError"),
|
|
28
|
+
):
|
|
29
|
+
if isinstance(error, kind):
|
|
30
|
+
return label
|
|
31
|
+
return "RuleError"
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class RecommendationType(str, Enum):
|
|
35
|
+
PARALLELIZE_TOOLS = "parallelize_tools"
|
|
36
|
+
CACHE_CONTEXT = "cache_context"
|
|
37
|
+
BATCH_MODEL_CALLS = "batch_model_calls"
|
|
38
|
+
ROUTE_TO_SMALLER_MODEL = "route_to_smaller_model"
|
|
39
|
+
REMOVE_RETRY_LOOP = "remove_retry_loop"
|
|
40
|
+
ADD_SCHEMA_VALIDATION = "add_schema_validation"
|
|
41
|
+
SPLIT_LARGE_STEP = "split_large_step"
|
|
42
|
+
RUNAWAY_LOOP = "runaway_loop"
|
|
43
|
+
TOOL_OSCILLATION = "tool_oscillation"
|
|
44
|
+
SEMANTIC_REDUNDANCY = "semantic_redundancy"
|
|
45
|
+
LOW_CONTRIBUTION = "low_contribution"
|
|
46
|
+
SEMANTIC_NO_PROGRESS = "semantic_no_progress"
|
|
47
|
+
RETRY_USEFULNESS = "retry_usefulness"
|
|
48
|
+
CONTEXT_RELEVANCE = "context_relevance"
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
@dataclass
|
|
52
|
+
class FindingCandidate:
|
|
53
|
+
type: RecommendationType
|
|
54
|
+
title: str
|
|
55
|
+
why: str
|
|
56
|
+
rewrite_hint: str
|
|
57
|
+
confidence: str
|
|
58
|
+
estimated_latency_savings_ms: float | None = 0.0
|
|
59
|
+
estimated_cost_savings_usd: float | None = 0.0
|
|
60
|
+
affected_nodes: list[str] | None = None
|
|
61
|
+
evidence_level: str | None = None
|
|
62
|
+
assumptions: list[str] | None = None
|
|
63
|
+
observations: dict[str, Any] | None = None
|
|
64
|
+
estimate_formula: str | None = None
|
|
65
|
+
|
|
66
|
+
rule_id: str | None = None
|
|
67
|
+
rule_version: str | None = None
|
|
68
|
+
estimate: dict[str, Any] | None = None
|
|
69
|
+
ranking_requirements: tuple[str, ...] = ()
|
|
70
|
+
|
|
71
|
+
def to_dict(self) -> dict[str, Any]:
|
|
72
|
+
result = {
|
|
73
|
+
"type": self.type.value,
|
|
74
|
+
"title": self.title,
|
|
75
|
+
"why": self.why,
|
|
76
|
+
"rewrite_hint": self.rewrite_hint,
|
|
77
|
+
"confidence": self.confidence,
|
|
78
|
+
"estimated_latency_savings_ms": None
|
|
79
|
+
if self.estimated_latency_savings_ms is None
|
|
80
|
+
else round(self.estimated_latency_savings_ms, 3),
|
|
81
|
+
"estimated_cost_savings_usd": (
|
|
82
|
+
None
|
|
83
|
+
if self.estimated_cost_savings_usd is None
|
|
84
|
+
else round(self.estimated_cost_savings_usd, 6)
|
|
85
|
+
),
|
|
86
|
+
"affected_nodes": self.affected_nodes or [],
|
|
87
|
+
}
|
|
88
|
+
if self.evidence_level is not None:
|
|
89
|
+
result.update(
|
|
90
|
+
evidence_level=self.evidence_level,
|
|
91
|
+
assumptions=self.assumptions or [],
|
|
92
|
+
observations=self.observations or {},
|
|
93
|
+
estimate_formula=self.estimate_formula,
|
|
94
|
+
)
|
|
95
|
+
if self.rule_id is not None:
|
|
96
|
+
result.update(rule_id=self.rule_id, rule_version=self.rule_version)
|
|
97
|
+
if self.estimate is not None:
|
|
98
|
+
result["estimate"] = deepcopy(self.estimate)
|
|
99
|
+
if self.ranking_requirements:
|
|
100
|
+
result["ranking_requirements"] = list(self.ranking_requirements)
|
|
101
|
+
return result
|
|
102
|
+
|
|
103
|
+
@classmethod
|
|
104
|
+
def from_dict(cls, data: dict[str, Any]) -> FindingCandidate:
|
|
105
|
+
values = {item.name: deepcopy(data[item.name]) for item in fields(cls) if item.name in data}
|
|
106
|
+
values["type"] = RecommendationType(values["type"])
|
|
107
|
+
return cls(**values)
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
@dataclass(frozen=True)
|
|
111
|
+
class AnalysisContext:
|
|
112
|
+
report: dict[str, Any]
|
|
113
|
+
graph: ExecutionGraph
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
@dataclass(frozen=True)
|
|
117
|
+
class FindingRule:
|
|
118
|
+
rule_id: str
|
|
119
|
+
version: str
|
|
120
|
+
detect: Callable[[AnalysisContext], list[FindingCandidate]]
|
|
121
|
+
ranking_requirements: tuple[str, ...] = ()
|
|
122
|
+
|
|
123
|
+
def __post_init__(self):
|
|
124
|
+
object.__setattr__(self, "ranking_requirements", requirements(self.ranking_requirements))
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def run_rules(
|
|
128
|
+
context: AnalysisContext, rules: tuple[FindingRule, ...] | None = None
|
|
129
|
+
) -> tuple[list[FindingCandidate], list[dict[str, str]]]:
|
|
130
|
+
"""Run canonical local rules in declared order, retaining failure diagnostics."""
|
|
131
|
+
candidates: list[FindingCandidate] = []
|
|
132
|
+
errors = []
|
|
133
|
+
for rule in BUILTIN_RULES if rules is None else rules:
|
|
134
|
+
try:
|
|
135
|
+
detected = list(rule.detect(context))
|
|
136
|
+
if any(not isinstance(item, FindingCandidate) for item in detected):
|
|
137
|
+
raise TypeError("rule must return FindingCandidate objects")
|
|
138
|
+
for item in detected:
|
|
139
|
+
item.rule_id = rule.rule_id
|
|
140
|
+
item.rule_version = rule.version
|
|
141
|
+
item.ranking_requirements = rule.ranking_requirements
|
|
142
|
+
if rule in BUILTIN_RULES and item.estimate is None:
|
|
143
|
+
item.estimate = estimate_snapshot(item, context.report, context.graph.nodes)
|
|
144
|
+
candidates.extend(detected)
|
|
145
|
+
except Exception as exc:
|
|
146
|
+
_LOGGER.debug("Finding rule %s failed", rule.rule_id, exc_info=True)
|
|
147
|
+
errors.append(
|
|
148
|
+
{
|
|
149
|
+
"rule_id": rule.rule_id,
|
|
150
|
+
"rule_version": rule.version,
|
|
151
|
+
"error_type": _safe_error_type(exc),
|
|
152
|
+
"message": "Rule evaluation failed; see local debug logs.",
|
|
153
|
+
}
|
|
154
|
+
)
|
|
155
|
+
if not is_cost_evaluable(context.report.get("cost_status", "complete")):
|
|
156
|
+
for candidate in candidates:
|
|
157
|
+
candidate.estimated_cost_savings_usd = None
|
|
158
|
+
from agentloop.interoperability.evidence import qualify_candidates
|
|
159
|
+
|
|
160
|
+
qualify_candidates(context.report, candidates)
|
|
161
|
+
return candidates, errors
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def _parallelization_cards(graph: ExecutionGraph) -> list[FindingCandidate]:
|
|
165
|
+
cards = []
|
|
166
|
+
for group in graph.parallelizable_groups():
|
|
167
|
+
cards.append(
|
|
168
|
+
FindingCandidate(
|
|
169
|
+
type=RecommendationType.PARALLELIZE_TOOLS,
|
|
170
|
+
title=f"Parallelize repeated `{group['name']}` tool calls",
|
|
171
|
+
why=f"{group['count']} repeated calls. {group['description']}",
|
|
172
|
+
rewrite_hint=PARALLELISM_REWRITE,
|
|
173
|
+
confidence=group["confidence"],
|
|
174
|
+
estimated_latency_savings_ms=group["estimated_savings_ms"],
|
|
175
|
+
affected_nodes=group["node_ids"],
|
|
176
|
+
evidence_level=group["evidence_level"],
|
|
177
|
+
assumptions=group["assumptions"],
|
|
178
|
+
observations=group["observations"],
|
|
179
|
+
estimate_formula=group["estimate_formula"],
|
|
180
|
+
)
|
|
181
|
+
)
|
|
182
|
+
return cards
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def _context_cache_cards(report: dict[str, Any], graph: ExecutionGraph) -> list[FindingCandidate]:
|
|
186
|
+
ratio = report.get("repeated_context_ratio", 0.0)
|
|
187
|
+
if ratio is None:
|
|
188
|
+
return []
|
|
189
|
+
if ratio < 0.10:
|
|
190
|
+
return []
|
|
191
|
+
model_nodes = [node.node_id for node in graph.nodes if node.operation_kind == "model"]
|
|
192
|
+
if not model_nodes:
|
|
193
|
+
return []
|
|
194
|
+
current_cost = report.get("estimated_cost_usd", 0.0)
|
|
195
|
+
cost_evaluable = is_cost_evaluable(report.get("cost_status", "complete"))
|
|
196
|
+
return [
|
|
197
|
+
FindingCandidate(
|
|
198
|
+
type=RecommendationType.CACHE_CONTEXT,
|
|
199
|
+
title="Cache repeated prompt/context prefix",
|
|
200
|
+
why=f"Repeated context ratio is {ratio:.1%}, which suggests stable instructions or source text are being resent.",
|
|
201
|
+
rewrite_hint="Move stable instructions into cached prefixes, summaries, or framework-level memory instead of resending full text.",
|
|
202
|
+
confidence="high" if ratio >= 0.20 else "medium",
|
|
203
|
+
estimated_cost_savings_usd=(
|
|
204
|
+
current_cost
|
|
205
|
+
* min(ESTIMATORS["cache_context"].parameter("max_cost_fraction"), ratio)
|
|
206
|
+
if cost_evaluable
|
|
207
|
+
else None
|
|
208
|
+
),
|
|
209
|
+
affected_nodes=model_nodes,
|
|
210
|
+
)
|
|
211
|
+
]
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def _retry_cards(graph: ExecutionGraph) -> list[FindingCandidate]:
|
|
215
|
+
retry_nodes = [node for node in graph.nodes if node.operation_kind == "retry"]
|
|
216
|
+
if not retry_nodes:
|
|
217
|
+
return []
|
|
218
|
+
retry_time = sum(node.duration_ms for node in retry_nodes)
|
|
219
|
+
return [
|
|
220
|
+
FindingCandidate(
|
|
221
|
+
type=RecommendationType.ADD_SCHEMA_VALIDATION,
|
|
222
|
+
title="Reduce retry loop with structured outputs",
|
|
223
|
+
why=f"Observed {len(retry_nodes)} retry span(s), costing {retry_time / 1000:.2f}s.",
|
|
224
|
+
rewrite_hint="Add schema validation, JSON mode, constrained decoding, or a cheap repair prompt before rerunning a full step.",
|
|
225
|
+
confidence="high",
|
|
226
|
+
estimated_latency_savings_ms=retry_time
|
|
227
|
+
* ESTIMATORS["add_schema_validation"].parameter("latency_fraction"),
|
|
228
|
+
affected_nodes=[node.node_id for node in retry_nodes],
|
|
229
|
+
)
|
|
230
|
+
]
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
def _batch_model_cards(graph: ExecutionGraph) -> list[FindingCandidate]:
|
|
234
|
+
cards = []
|
|
235
|
+
for name, nodes in model_batch_groups(graph):
|
|
236
|
+
duration = sum(node.duration_ms for node in nodes)
|
|
237
|
+
cards.append(
|
|
238
|
+
FindingCandidate(
|
|
239
|
+
type=RecommendationType.BATCH_MODEL_CALLS,
|
|
240
|
+
title=f"Batch repeated `{name}` model calls",
|
|
241
|
+
why=f"{len(nodes)} `{name}` calls have the same role and may be batchable.",
|
|
242
|
+
rewrite_hint="Batch documents/items into one prompt or use map-reduce only when outputs truly need independent reasoning.",
|
|
243
|
+
confidence="medium",
|
|
244
|
+
estimated_latency_savings_ms=duration
|
|
245
|
+
* ESTIMATORS["batch_model_calls"].parameter("latency_fraction"),
|
|
246
|
+
affected_nodes=[node.node_id for node in nodes],
|
|
247
|
+
)
|
|
248
|
+
)
|
|
249
|
+
return cards
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def _routing_cards(graph: ExecutionGraph) -> list[FindingCandidate]:
|
|
253
|
+
cards = []
|
|
254
|
+
for node in graph.nodes:
|
|
255
|
+
if node.operation_kind != "model" or not node.model:
|
|
256
|
+
continue
|
|
257
|
+
if node.token_provenance == UNAVAILABLE or (
|
|
258
|
+
isinstance(node.metadata, Mapping) and node.metadata.get("usage_available") is False
|
|
259
|
+
):
|
|
260
|
+
continue
|
|
261
|
+
if "mini" in node.model.lower():
|
|
262
|
+
continue
|
|
263
|
+
if node.output_tokens <= 400 and node.input_tokens <= 2500:
|
|
264
|
+
cards.append(
|
|
265
|
+
FindingCandidate(
|
|
266
|
+
type=RecommendationType.ROUTE_TO_SMALLER_MODEL,
|
|
267
|
+
title=f"Route `{node.name}` to a cheaper model",
|
|
268
|
+
why=f"`{node.name}` is a relatively small model step using {node.total_tokens} tokens on {node.model}.",
|
|
269
|
+
rewrite_hint="Try a smaller model for planning, summarization, extraction, or verification steps and keep the larger model for final synthesis.",
|
|
270
|
+
confidence="low",
|
|
271
|
+
estimated_latency_savings_ms=node.duration_ms
|
|
272
|
+
* ESTIMATORS["route_to_smaller_model"].parameter("latency_fraction"),
|
|
273
|
+
affected_nodes=[node.node_id],
|
|
274
|
+
)
|
|
275
|
+
)
|
|
276
|
+
return cards[:3]
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
def _split_large_step_cards(graph: ExecutionGraph) -> list[FindingCandidate]:
|
|
280
|
+
cards = []
|
|
281
|
+
for node in graph.nodes:
|
|
282
|
+
if node.operation_kind == "model" and node.total_tokens >= 4000:
|
|
283
|
+
cards.append(
|
|
284
|
+
FindingCandidate(
|
|
285
|
+
type=RecommendationType.SPLIT_LARGE_STEP,
|
|
286
|
+
title=f"Split or compress `{node.name}`",
|
|
287
|
+
why=f"`{node.name}` used {node.total_tokens} tokens, making it a large and fragile step.",
|
|
288
|
+
rewrite_hint="Split into retrieve-filter-summarize or compress the context before the final reasoning call.",
|
|
289
|
+
confidence="medium",
|
|
290
|
+
estimated_latency_savings_ms=node.duration_ms
|
|
291
|
+
* ESTIMATORS["split_large_step"].parameter("latency_fraction"),
|
|
292
|
+
affected_nodes=[node.node_id],
|
|
293
|
+
)
|
|
294
|
+
)
|
|
295
|
+
return cards
|
|
296
|
+
|
|
297
|
+
|
|
298
|
+
def _runaway_loop_cards(graph: ExecutionGraph) -> list[FindingCandidate]:
|
|
299
|
+
if graph.execution is not None:
|
|
300
|
+
return [] # Repeated pipeline stages do not establish an agent loop.
|
|
301
|
+
cards = []
|
|
302
|
+
groups: dict[tuple[str, str], list[Any]] = {}
|
|
303
|
+
for node in graph.nodes:
|
|
304
|
+
groups.setdefault((node.operation_kind, node.name), []).append(node)
|
|
305
|
+
|
|
306
|
+
for (event_type, name), nodes in groups.items():
|
|
307
|
+
if len(nodes) < 8:
|
|
308
|
+
continue
|
|
309
|
+
duration = sum(node.duration_ms for node in nodes)
|
|
310
|
+
cards.append(
|
|
311
|
+
FindingCandidate(
|
|
312
|
+
type=RecommendationType.RUNAWAY_LOOP,
|
|
313
|
+
title=f"Add a loop guardrail around `{name}`",
|
|
314
|
+
why=f"`{name}` ran {len(nodes)} times in one trace, which suggests an unbounded or weakly bounded agent loop.",
|
|
315
|
+
rewrite_hint="Add max-iteration, max-cost, and unchanged-state guards before the workflow can keep looping.",
|
|
316
|
+
confidence="high" if len(nodes) >= 12 else "medium",
|
|
317
|
+
estimated_latency_savings_ms=duration
|
|
318
|
+
* ESTIMATORS["runaway_loop"].parameter("latency_fraction"),
|
|
319
|
+
affected_nodes=[node.node_id for node in nodes],
|
|
320
|
+
)
|
|
321
|
+
)
|
|
322
|
+
return cards[:3]
|
|
323
|
+
|
|
324
|
+
|
|
325
|
+
def _tool_oscillation_cards(graph: ExecutionGraph) -> list[FindingCandidate]:
|
|
326
|
+
if graph.execution is not None:
|
|
327
|
+
return [] # Generic workflow ordering is not evidence of agent oscillation.
|
|
328
|
+
tool_nodes = [node for node in graph.nodes if node.operation_kind == "tool"]
|
|
329
|
+
if len(tool_nodes) < 4:
|
|
330
|
+
return []
|
|
331
|
+
|
|
332
|
+
cards = []
|
|
333
|
+
index = 0
|
|
334
|
+
while index <= len(tool_nodes) - 4:
|
|
335
|
+
first = tool_nodes[index].name
|
|
336
|
+
second = tool_nodes[index + 1].name
|
|
337
|
+
if first == second:
|
|
338
|
+
index += 1
|
|
339
|
+
continue
|
|
340
|
+
|
|
341
|
+
window = tool_nodes[index : index + 4]
|
|
342
|
+
if [node.name for node in window] != [first, second, first, second]:
|
|
343
|
+
index += 1
|
|
344
|
+
continue
|
|
345
|
+
|
|
346
|
+
end = index + 4
|
|
347
|
+
while end < len(tool_nodes) and tool_nodes[end].name == (
|
|
348
|
+
first if (end - index) % 2 == 0 else second
|
|
349
|
+
):
|
|
350
|
+
end += 1
|
|
351
|
+
oscillating = tool_nodes[index:end]
|
|
352
|
+
duration = sum(node.duration_ms for node in oscillating)
|
|
353
|
+
cards.append(
|
|
354
|
+
FindingCandidate(
|
|
355
|
+
type=RecommendationType.TOOL_OSCILLATION,
|
|
356
|
+
title=f"Stop `{first}`/`{second}` tool oscillation",
|
|
357
|
+
why=f"Observed {len(oscillating)} alternating `{first}` and `{second}` tool calls.",
|
|
358
|
+
rewrite_hint="Add a state-change check or decision memo so the agent does not repeat equivalent tool transitions.",
|
|
359
|
+
confidence="medium",
|
|
360
|
+
estimated_latency_savings_ms=duration
|
|
361
|
+
* ESTIMATORS["tool_oscillation"].parameter("latency_fraction"),
|
|
362
|
+
affected_nodes=[node.node_id for node in oscillating],
|
|
363
|
+
)
|
|
364
|
+
)
|
|
365
|
+
index = end
|
|
366
|
+
|
|
367
|
+
return sorted(cards, key=lambda card: card.estimated_latency_savings_ms, reverse=True)[:3]
|
|
368
|
+
|
|
369
|
+
|
|
370
|
+
def _semantic_cards(context, family):
|
|
371
|
+
if not context.report.get("semantic_waste"):
|
|
372
|
+
return []
|
|
373
|
+
from agentloop.semantic_rules import semantic_candidates
|
|
374
|
+
|
|
375
|
+
return semantic_candidates(context, family)
|
|
376
|
+
|
|
377
|
+
|
|
378
|
+
BUILTIN_RULES = (
|
|
379
|
+
FindingRule("parallelize_tools", "1.1", lambda ctx: _parallelization_cards(ctx.graph)),
|
|
380
|
+
FindingRule("cache_context", "1.1", lambda ctx: _context_cache_cards(ctx.report, ctx.graph)),
|
|
381
|
+
FindingRule("add_schema_validation", "1.1", lambda ctx: _retry_cards(ctx.graph)),
|
|
382
|
+
FindingRule("batch_model_calls", "1.2", lambda ctx: _batch_model_cards(ctx.graph)),
|
|
383
|
+
FindingRule("route_to_smaller_model", "1.2", lambda ctx: _routing_cards(ctx.graph)),
|
|
384
|
+
FindingRule("split_large_step", "1.1", lambda ctx: _split_large_step_cards(ctx.graph)),
|
|
385
|
+
FindingRule("runaway_loop", "1.1", lambda ctx: _runaway_loop_cards(ctx.graph)),
|
|
386
|
+
FindingRule("tool_oscillation", "1.1", lambda ctx: _tool_oscillation_cards(ctx.graph)),
|
|
387
|
+
FindingRule(
|
|
388
|
+
"semantic_redundancy", "1.0", lambda ctx: _semantic_cards(ctx, "semantic_redundancy")
|
|
389
|
+
),
|
|
390
|
+
FindingRule("low_contribution", "1.0", lambda ctx: _semantic_cards(ctx, "low_contribution")),
|
|
391
|
+
FindingRule(
|
|
392
|
+
"semantic_no_progress", "1.0", lambda ctx: _semantic_cards(ctx, "semantic_no_progress")
|
|
393
|
+
),
|
|
394
|
+
FindingRule("retry_usefulness", "1.0", lambda ctx: _semantic_cards(ctx, "retry_usefulness")),
|
|
395
|
+
FindingRule("context_relevance", "1.0", lambda ctx: _semantic_cards(ctx, "context_relevance")),
|
|
396
|
+
)
|
agentloop/runtime.py
ADDED
|
@@ -0,0 +1,286 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
from dataclasses import dataclass
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import TYPE_CHECKING, Any
|
|
7
|
+
|
|
8
|
+
if TYPE_CHECKING:
|
|
9
|
+
from agentloop.retention import RetentionSession
|
|
10
|
+
|
|
11
|
+
_TRUE_VALUES = {"1", "true", "yes", "on"}
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class _ClearSentinel:
|
|
15
|
+
"""Sentinel passed to :func:`init` to explicitly clear an optional value.
|
|
16
|
+
|
|
17
|
+
``init(api_key=None)`` means "keep the current value" (so environment and
|
|
18
|
+
prior configuration survive a reconfigure). Passing ``CLEAR`` instead sets
|
|
19
|
+
the value back to ``None``.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
def __repr__(self) -> str:
|
|
23
|
+
return "CLEAR"
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
CLEAR = _ClearSentinel()
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class FinalizationError(RuntimeError):
|
|
30
|
+
"""Raised by :func:`finalize_trace` when a destination fails and
|
|
31
|
+
``fail_silently`` is ``False``.
|
|
32
|
+
|
|
33
|
+
``result`` holds the partial finalization result for destinations that
|
|
34
|
+
completed before the failure, so already-completed work is not lost.
|
|
35
|
+
``errors`` lists the destination-tagged failures.
|
|
36
|
+
"""
|
|
37
|
+
|
|
38
|
+
def __init__(
|
|
39
|
+
self,
|
|
40
|
+
message: str,
|
|
41
|
+
*,
|
|
42
|
+
result: dict[str, Any],
|
|
43
|
+
errors: list[dict[str, str]],
|
|
44
|
+
) -> None:
|
|
45
|
+
super().__init__(message)
|
|
46
|
+
self.result = result
|
|
47
|
+
self.errors = errors
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
@dataclass
|
|
51
|
+
class AgentLoopRuntimeConfig:
|
|
52
|
+
api_url: str = "http://127.0.0.1:8000"
|
|
53
|
+
api_key: str | None = None
|
|
54
|
+
project_id: str = "default"
|
|
55
|
+
auto_upload: bool = False
|
|
56
|
+
auto_store: bool = False
|
|
57
|
+
fail_silently: bool = True
|
|
58
|
+
export_dir: Path | None = None
|
|
59
|
+
retention: RetentionSession | None = None
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
_runtime = AgentLoopRuntimeConfig()
|
|
63
|
+
_last_error: str | None = None
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _env_bool(name: str, default: bool = False) -> bool:
|
|
67
|
+
value = os.getenv(name)
|
|
68
|
+
if value is None:
|
|
69
|
+
return default
|
|
70
|
+
return value.lower() in _TRUE_VALUES
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def init(
|
|
74
|
+
*,
|
|
75
|
+
api_url: str | None = None,
|
|
76
|
+
api_key: str | _ClearSentinel | None = None,
|
|
77
|
+
project_id: str | None = None,
|
|
78
|
+
auto_upload: bool | None = None,
|
|
79
|
+
auto_store: bool | None = None,
|
|
80
|
+
fail_silently: bool | None = None,
|
|
81
|
+
export_dir: str | Path | _ClearSentinel | None = None,
|
|
82
|
+
retention: RetentionSession | _ClearSentinel | None = None,
|
|
83
|
+
) -> AgentLoopRuntimeConfig:
|
|
84
|
+
"""Configure AgentLoop once at process startup.
|
|
85
|
+
|
|
86
|
+
Examples:
|
|
87
|
+
|
|
88
|
+
```python
|
|
89
|
+
import agentloop
|
|
90
|
+
|
|
91
|
+
agentloop.init(api_key="al_xxx", project_id="demo", auto_upload=True)
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
Environment variables are also supported:
|
|
95
|
+
- `AGENTLOOP_API_URL`
|
|
96
|
+
- `AGENTLOOP_API_KEY`
|
|
97
|
+
- `AGENTLOOP_PROJECT_ID`
|
|
98
|
+
- `AGENTLOOP_AUTO_UPLOAD`
|
|
99
|
+
- `AGENTLOOP_AUTO_STORE`
|
|
100
|
+
- `AGENTLOOP_FAIL_SILENTLY`
|
|
101
|
+
- `AGENTLOOP_EXPORT_DIR`
|
|
102
|
+
|
|
103
|
+
Optional values (`api_key`, `export_dir`) treat ``None`` as "keep the current
|
|
104
|
+
value". Pass :data:`CLEAR` to reset one of them back to ``None``.
|
|
105
|
+
"""
|
|
106
|
+
|
|
107
|
+
global _runtime
|
|
108
|
+
|
|
109
|
+
from agentloop.retention import RetentionSession
|
|
110
|
+
|
|
111
|
+
if (
|
|
112
|
+
retention is not None
|
|
113
|
+
and retention is not CLEAR
|
|
114
|
+
and not isinstance(retention, RetentionSession)
|
|
115
|
+
):
|
|
116
|
+
raise TypeError("retention must be RetentionSession, CLEAR or None")
|
|
117
|
+
|
|
118
|
+
if api_key is CLEAR:
|
|
119
|
+
resolved_api_key: str | None = None
|
|
120
|
+
elif api_key is not None:
|
|
121
|
+
resolved_api_key = api_key # type: ignore[assignment]
|
|
122
|
+
else:
|
|
123
|
+
resolved_api_key = os.getenv("AGENTLOOP_API_KEY", _runtime.api_key)
|
|
124
|
+
|
|
125
|
+
if export_dir is CLEAR:
|
|
126
|
+
resolved_export_dir: Path | None = None
|
|
127
|
+
elif export_dir is not None:
|
|
128
|
+
resolved_export_dir = Path(export_dir) # type: ignore[arg-type]
|
|
129
|
+
else:
|
|
130
|
+
env_export_dir = os.getenv("AGENTLOOP_EXPORT_DIR")
|
|
131
|
+
resolved_export_dir = Path(env_export_dir) if env_export_dir else _runtime.export_dir
|
|
132
|
+
|
|
133
|
+
_runtime = AgentLoopRuntimeConfig(
|
|
134
|
+
api_url=api_url or os.getenv("AGENTLOOP_API_URL", _runtime.api_url),
|
|
135
|
+
api_key=resolved_api_key,
|
|
136
|
+
project_id=project_id or os.getenv("AGENTLOOP_PROJECT_ID", _runtime.project_id),
|
|
137
|
+
auto_upload=auto_upload
|
|
138
|
+
if auto_upload is not None
|
|
139
|
+
else _env_bool("AGENTLOOP_AUTO_UPLOAD", _runtime.auto_upload),
|
|
140
|
+
auto_store=auto_store
|
|
141
|
+
if auto_store is not None
|
|
142
|
+
else _env_bool("AGENTLOOP_AUTO_STORE", _runtime.auto_store),
|
|
143
|
+
fail_silently=fail_silently
|
|
144
|
+
if fail_silently is not None
|
|
145
|
+
else _env_bool("AGENTLOOP_FAIL_SILENTLY", _runtime.fail_silently),
|
|
146
|
+
export_dir=resolved_export_dir,
|
|
147
|
+
retention=None
|
|
148
|
+
if retention is CLEAR
|
|
149
|
+
else retention
|
|
150
|
+
if retention is not None
|
|
151
|
+
else _runtime.retention,
|
|
152
|
+
)
|
|
153
|
+
return _runtime
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def configure_from_env() -> AgentLoopRuntimeConfig:
|
|
157
|
+
return init()
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def get_runtime_config() -> AgentLoopRuntimeConfig:
|
|
161
|
+
return _runtime
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def get_last_error() -> str | None:
|
|
165
|
+
return _last_error
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def reset_runtime() -> None:
|
|
169
|
+
global _runtime, _last_error
|
|
170
|
+
_runtime = AgentLoopRuntimeConfig()
|
|
171
|
+
_last_error = None
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def should_auto_export() -> bool:
|
|
175
|
+
return bool(_runtime.auto_upload or _runtime.auto_store or _runtime.export_dir)
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def finalize_trace(trace: Any) -> dict[str, Any]:
|
|
179
|
+
"""Apply configured end-of-run side effects.
|
|
180
|
+
|
|
181
|
+
Depending on runtime config, this exports a JSON file, saves to the local
|
|
182
|
+
persistent store, and/or uploads to the configured API. Each destination runs
|
|
183
|
+
in its own error boundary in a deterministic order (export, store, upload), so
|
|
184
|
+
a failure in one cannot prevent the others from running.
|
|
185
|
+
|
|
186
|
+
When ``fail_silently`` is ``True`` (the default), every configured destination
|
|
187
|
+
is attempted and failures are collected in ``result["errors"]``, each tagged
|
|
188
|
+
with its ``destination``. When ``fail_silently`` is ``False``, the first failing
|
|
189
|
+
destination raises :class:`FinalizationError` (fail-fast); its ``result``
|
|
190
|
+
attribute still carries the destinations that completed before the failure.
|
|
191
|
+
|
|
192
|
+
A finalization in which every attempted destination succeeds clears any stale
|
|
193
|
+
process-global error so :func:`get_last_error` reflects the latest outcome.
|
|
194
|
+
"""
|
|
195
|
+
|
|
196
|
+
global _last_error
|
|
197
|
+
result: dict[str, Any] = {
|
|
198
|
+
"exported_path": None,
|
|
199
|
+
"stored": False,
|
|
200
|
+
"uploaded": False,
|
|
201
|
+
"errors": [],
|
|
202
|
+
}
|
|
203
|
+
if _runtime.retention is not None:
|
|
204
|
+
try:
|
|
205
|
+
trace = _runtime.retention.retain(trace)
|
|
206
|
+
result["retention"] = {
|
|
207
|
+
"retained": trace is not None,
|
|
208
|
+
"sampling": _runtime.retention.summary(),
|
|
209
|
+
}
|
|
210
|
+
except Exception as exc:
|
|
211
|
+
# Never fall back to raw export after a privacy/retention failure.
|
|
212
|
+
errors = [
|
|
213
|
+
{
|
|
214
|
+
"destination": "retention",
|
|
215
|
+
"error": "Retention failed; no destinations attempted.",
|
|
216
|
+
}
|
|
217
|
+
]
|
|
218
|
+
result["errors"] = errors
|
|
219
|
+
_last_error = errors[0]["error"]
|
|
220
|
+
if not _runtime.fail_silently:
|
|
221
|
+
raise FinalizationError(_last_error, result=result, errors=errors) from exc
|
|
222
|
+
return result
|
|
223
|
+
if trace is None:
|
|
224
|
+
_last_error = None
|
|
225
|
+
return result
|
|
226
|
+
|
|
227
|
+
def _do_export() -> None:
|
|
228
|
+
export_dir = _runtime.export_dir
|
|
229
|
+
if export_dir is None: # pragma: no cover - guarded before registration
|
|
230
|
+
return
|
|
231
|
+
export_dir.mkdir(parents=True, exist_ok=True)
|
|
232
|
+
out = export_dir / f"{trace.run_id}.json"
|
|
233
|
+
trace.export_json(out)
|
|
234
|
+
result["exported_path"] = str(out)
|
|
235
|
+
|
|
236
|
+
def _do_store() -> None:
|
|
237
|
+
from agentloop.store import get_store
|
|
238
|
+
|
|
239
|
+
db = get_store()
|
|
240
|
+
db.init()
|
|
241
|
+
db.save_trace(trace, project_id=_runtime.project_id)
|
|
242
|
+
result["stored"] = True
|
|
243
|
+
|
|
244
|
+
def _do_upload() -> None:
|
|
245
|
+
from agentloop.client import AgentLoopClient
|
|
246
|
+
|
|
247
|
+
client = AgentLoopClient(base_url=_runtime.api_url, api_key=_runtime.api_key)
|
|
248
|
+
result["upload_response"] = client.upload_trace(trace)
|
|
249
|
+
result["uploaded"] = True
|
|
250
|
+
|
|
251
|
+
destinations: list[tuple[str, Any]] = []
|
|
252
|
+
if _runtime.export_dir is not None:
|
|
253
|
+
destinations.append(("export", _do_export))
|
|
254
|
+
if _runtime.auto_store:
|
|
255
|
+
destinations.append(("store", _do_store))
|
|
256
|
+
if _runtime.auto_upload:
|
|
257
|
+
destinations.append(("upload", _do_upload))
|
|
258
|
+
|
|
259
|
+
errors: list[dict[str, str]] = []
|
|
260
|
+
for destination, action in destinations:
|
|
261
|
+
try:
|
|
262
|
+
action()
|
|
263
|
+
except Exception as exc:
|
|
264
|
+
errors.append({"destination": destination, "error": str(exc)})
|
|
265
|
+
if not _runtime.fail_silently:
|
|
266
|
+
result["errors"] = errors
|
|
267
|
+
_last_error = f"{destination}: {exc}"
|
|
268
|
+
raise FinalizationError(
|
|
269
|
+
f"{destination} finalization failed: {exc}",
|
|
270
|
+
result=result,
|
|
271
|
+
errors=errors,
|
|
272
|
+
) from exc
|
|
273
|
+
|
|
274
|
+
result["errors"] = errors
|
|
275
|
+
if errors:
|
|
276
|
+
last = errors[-1]
|
|
277
|
+
_last_error = f"{last['destination']}: {last['error']}"
|
|
278
|
+
elif destinations:
|
|
279
|
+
# Every attempted destination succeeded; drop any stale error state.
|
|
280
|
+
_last_error = None
|
|
281
|
+
|
|
282
|
+
return result
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
# Load env-driven defaults on import without forcing users to call init().
|
|
286
|
+
configure_from_env()
|