agentloop-profiler 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (166) hide show
  1. agentloop/__init__.py +136 -0
  2. agentloop/ablation_protocol.py +229 -0
  3. agentloop/ablations.py +564 -0
  4. agentloop/aggregate_cli.py +88 -0
  5. agentloop/aggregate_stats.py +175 -0
  6. agentloop/aggregates.py +532 -0
  7. agentloop/audit.py +39 -0
  8. agentloop/autoinstrument.py +145 -0
  9. agentloop/batch_experiments.py +295 -0
  10. agentloop/batching.py +95 -0
  11. agentloop/budget_types.py +335 -0
  12. agentloop/budgets.py +539 -0
  13. agentloop/calibration.py +872 -0
  14. agentloop/checkpoint_state.py +306 -0
  15. agentloop/checkpoint_store.py +86 -0
  16. agentloop/ci.py +202 -0
  17. agentloop/cli.py +1314 -0
  18. agentloop/client.py +228 -0
  19. agentloop/completion.py +345 -0
  20. agentloop/completion_types.py +199 -0
  21. agentloop/config.py +50 -0
  22. agentloop/context_controls.py +443 -0
  23. agentloop/context_experiments.py +337 -0
  24. agentloop/context_types.py +216 -0
  25. agentloop/costs.py +667 -0
  26. agentloop/cursor_validation.py +48 -0
  27. agentloop/decorators.py +445 -0
  28. agentloop/demo.py +300 -0
  29. agentloop/doctor.py +341 -0
  30. agentloop/drift.py +361 -0
  31. agentloop/drift_cli.py +147 -0
  32. agentloop/drift_types.py +110 -0
  33. agentloop/drift_windows.py +340 -0
  34. agentloop/entrypoint.py +201 -0
  35. agentloop/estimates.py +133 -0
  36. agentloop/events.py +75 -0
  37. agentloop/experiment_artifacts.py +80 -0
  38. agentloop/experiment_observations.py +169 -0
  39. agentloop/experiment_reports.py +545 -0
  40. agentloop/experiment_types.py +355 -0
  41. agentloop/experiments.py +392 -0
  42. agentloop/exporters.py +117 -0
  43. agentloop/finding_benchmark_types.py +131 -0
  44. agentloop/finding_benchmarks.py +424 -0
  45. agentloop/findings.py +419 -0
  46. agentloop/graph.py +464 -0
  47. agentloop/harness.py +825 -0
  48. agentloop/harness_bench/__init__.py +1 -0
  49. agentloop/harness_bench/cli.py +77 -0
  50. agentloop/harness_bench/drivers.py +96 -0
  51. agentloop/harness_bench/offline.py +507 -0
  52. agentloop/harness_evidence.py +585 -0
  53. agentloop/harness_usage.py +91 -0
  54. agentloop/html_report.py +860 -0
  55. agentloop/integrations/__init__.py +1 -0
  56. agentloop/integrations/crewai.py +144 -0
  57. agentloop/integrations/harbor/__init__.py +1 -0
  58. agentloop/integrations/harbor/atif.py +982 -0
  59. agentloop/integrations/harbor/atif_validation.py +329 -0
  60. agentloop/integrations/harbor/cli.py +207 -0
  61. agentloop/integrations/harbor/manifest.py +86 -0
  62. agentloop/integrations/harbor/trial_evidence.py +196 -0
  63. agentloop/integrations/harbor/trials.py +1109 -0
  64. agentloop/integrations/harbor/verifier.py +118 -0
  65. agentloop/integrations/langgraph.py +150 -0
  66. agentloop/integrations/langgraph_harness.py +466 -0
  67. agentloop/integrations/langgraph_recovery.py +366 -0
  68. agentloop/integrations/omnigent/__init__.py +1 -0
  69. agentloop/integrations/omnigent/cli.py +78 -0
  70. agentloop/integrations/omnigent/evidence.py +93 -0
  71. agentloop/integrations/omnigent/telemetry.py +422 -0
  72. agentloop/integrations/openai.py +415 -0
  73. agentloop/integrations/openai_agents.py +291 -0
  74. agentloop/integrations/sdk.py +82 -0
  75. agentloop/integrations/vercel_ai.py +83 -0
  76. agentloop/interoperability/__init__.py +5 -0
  77. agentloop/interoperability/artifacts.py +112 -0
  78. agentloop/interoperability/cli.py +59 -0
  79. agentloop/interoperability/cohort_evidence.py +175 -0
  80. agentloop/interoperability/cohorts.py +681 -0
  81. agentloop/interoperability/contracts.py +427 -0
  82. agentloop/interoperability/coordination.py +554 -0
  83. agentloop/interoperability/evidence.py +215 -0
  84. agentloop/interoperability/otlp_jsonl.py +775 -0
  85. agentloop/interoperability/otlp_validation.py +496 -0
  86. agentloop/interoperability/registry.py +561 -0
  87. agentloop/interoperability/validation.py +333 -0
  88. agentloop/intervention_service.py +110 -0
  89. agentloop/intervention_store.py +118 -0
  90. agentloop/interventions.py +190 -0
  91. agentloop/issues.py +90 -0
  92. agentloop/judgment_adapters.py +125 -0
  93. agentloop/judgment_benchmark_types.py +98 -0
  94. agentloop/judgment_benchmarks.py +606 -0
  95. agentloop/judgment_types.py +241 -0
  96. agentloop/judgment_views.py +38 -0
  97. agentloop/judgments.py +342 -0
  98. agentloop/loop_guards.py +274 -0
  99. agentloop/loop_types.py +66 -0
  100. agentloop/markdown.py +102 -0
  101. agentloop/metrics.py +342 -0
  102. agentloop/migrations.py +303 -0
  103. agentloop/model_routing.py +470 -0
  104. agentloop/onboarding.py +315 -0
  105. agentloop/operations.py +44 -0
  106. agentloop/optimizer.py +199 -0
  107. agentloop/otel.py +715 -0
  108. agentloop/otel_ids.py +81 -0
  109. agentloop/otel_semantics.py +134 -0
  110. agentloop/parallelism.py +141 -0
  111. agentloop/patches.py +603 -0
  112. agentloop/plan_export.py +126 -0
  113. agentloop/policy_promotion.py +325 -0
  114. agentloop/promotion_evidence.py +304 -0
  115. agentloop/promotion_types.py +209 -0
  116. agentloop/quality.py +446 -0
  117. agentloop/ranking.py +529 -0
  118. agentloop/recovery_evidence.py +96 -0
  119. agentloop/replay.py +753 -0
  120. agentloop/retention.py +375 -0
  121. agentloop/retention_types.py +113 -0
  122. agentloop/reuse_experiments.py +247 -0
  123. agentloop/routing_checks.py +116 -0
  124. agentloop/routing_types.py +241 -0
  125. agentloop/rules.py +396 -0
  126. agentloop/runtime.py +286 -0
  127. agentloop/savings.py +158 -0
  128. agentloop/scheduling_plan.py +99 -0
  129. agentloop/scheduling_types.py +169 -0
  130. agentloop/schema.py +257 -0
  131. agentloop/semantic_rules.py +200 -0
  132. agentloop/semantic_waste.py +345 -0
  133. agentloop/semantic_waste_types.py +88 -0
  134. agentloop/server.py +494 -0
  135. agentloop/store.py +1429 -0
  136. agentloop/structural_experiments.py +431 -0
  137. agentloop/structural_types.py +113 -0
  138. agentloop/structured_quality.py +1020 -0
  139. agentloop/studies.py +629 -0
  140. agentloop/study_statistics.py +58 -0
  141. agentloop/substitution_evidence.py +199 -0
  142. agentloop/substitution_export.py +190 -0
  143. agentloop/substitution_reports.py +407 -0
  144. agentloop/substitution_types.py +284 -0
  145. agentloop/substitutions.py +267 -0
  146. agentloop/timing.py +142 -0
  147. agentloop/tokens.py +188 -0
  148. agentloop/tool_scheduling.py +410 -0
  149. agentloop/trace_sources.py +31 -0
  150. agentloop/tracer.py +494 -0
  151. agentloop/value.py +306 -0
  152. agentloop/version.py +3 -0
  153. agentloop/workflow_types.py +264 -0
  154. agentloop/workflows.py +284 -0
  155. agentloop_profiler-0.1.0.dist-info/METADATA +513 -0
  156. agentloop_profiler-0.1.0.dist-info/RECORD +166 -0
  157. agentloop_profiler-0.1.0.dist-info/WHEEL +5 -0
  158. agentloop_profiler-0.1.0.dist-info/entry_points.txt +2 -0
  159. agentloop_profiler-0.1.0.dist-info/licenses/LICENSE +201 -0
  160. agentloop_profiler-0.1.0.dist-info/licenses/THIRD_PARTY_LICENSES.md +101 -0
  161. agentloop_profiler-0.1.0.dist-info/top_level.txt +2 -0
  162. dashboard/__init__.py +5 -0
  163. dashboard/app.py +922 -0
  164. dashboard/streamlit_compat.py +63 -0
  165. dashboard/trace_upload.py +57 -0
  166. dashboard/value_view.py +105 -0
agentloop/rules.py ADDED
@@ -0,0 +1,396 @@
1
+ from __future__ import annotations
2
+
3
+ import logging
4
+ from collections.abc import Callable, Mapping
5
+ from copy import deepcopy
6
+ from dataclasses import dataclass, fields
7
+ from enum import Enum
8
+ from typing import Any
9
+
10
+ from agentloop.batching import model_batch_groups
11
+ from agentloop.costs import is_cost_evaluable
12
+ from agentloop.estimates import ESTIMATORS, estimate_snapshot
13
+ from agentloop.graph import ExecutionGraph
14
+ from agentloop.parallelism import PARALLELISM_REWRITE
15
+ from agentloop.ranking import requirements
16
+ from agentloop.tokens import UNAVAILABLE
17
+
18
+ _LOGGER = logging.getLogger(__name__)
19
+
20
+
21
+ def _safe_error_type(error: Exception) -> str:
22
+ """Use public categories rather than exposing arbitrary implementation class names."""
23
+ for kind, label in (
24
+ (TypeError, "TypeError"),
25
+ (ValueError, "ValueError"),
26
+ (LookupError, "LookupError"),
27
+ (ArithmeticError, "ArithmeticError"),
28
+ ):
29
+ if isinstance(error, kind):
30
+ return label
31
+ return "RuleError"
32
+
33
+
34
+ class RecommendationType(str, Enum):
35
+ PARALLELIZE_TOOLS = "parallelize_tools"
36
+ CACHE_CONTEXT = "cache_context"
37
+ BATCH_MODEL_CALLS = "batch_model_calls"
38
+ ROUTE_TO_SMALLER_MODEL = "route_to_smaller_model"
39
+ REMOVE_RETRY_LOOP = "remove_retry_loop"
40
+ ADD_SCHEMA_VALIDATION = "add_schema_validation"
41
+ SPLIT_LARGE_STEP = "split_large_step"
42
+ RUNAWAY_LOOP = "runaway_loop"
43
+ TOOL_OSCILLATION = "tool_oscillation"
44
+ SEMANTIC_REDUNDANCY = "semantic_redundancy"
45
+ LOW_CONTRIBUTION = "low_contribution"
46
+ SEMANTIC_NO_PROGRESS = "semantic_no_progress"
47
+ RETRY_USEFULNESS = "retry_usefulness"
48
+ CONTEXT_RELEVANCE = "context_relevance"
49
+
50
+
51
+ @dataclass
52
+ class FindingCandidate:
53
+ type: RecommendationType
54
+ title: str
55
+ why: str
56
+ rewrite_hint: str
57
+ confidence: str
58
+ estimated_latency_savings_ms: float | None = 0.0
59
+ estimated_cost_savings_usd: float | None = 0.0
60
+ affected_nodes: list[str] | None = None
61
+ evidence_level: str | None = None
62
+ assumptions: list[str] | None = None
63
+ observations: dict[str, Any] | None = None
64
+ estimate_formula: str | None = None
65
+
66
+ rule_id: str | None = None
67
+ rule_version: str | None = None
68
+ estimate: dict[str, Any] | None = None
69
+ ranking_requirements: tuple[str, ...] = ()
70
+
71
+ def to_dict(self) -> dict[str, Any]:
72
+ result = {
73
+ "type": self.type.value,
74
+ "title": self.title,
75
+ "why": self.why,
76
+ "rewrite_hint": self.rewrite_hint,
77
+ "confidence": self.confidence,
78
+ "estimated_latency_savings_ms": None
79
+ if self.estimated_latency_savings_ms is None
80
+ else round(self.estimated_latency_savings_ms, 3),
81
+ "estimated_cost_savings_usd": (
82
+ None
83
+ if self.estimated_cost_savings_usd is None
84
+ else round(self.estimated_cost_savings_usd, 6)
85
+ ),
86
+ "affected_nodes": self.affected_nodes or [],
87
+ }
88
+ if self.evidence_level is not None:
89
+ result.update(
90
+ evidence_level=self.evidence_level,
91
+ assumptions=self.assumptions or [],
92
+ observations=self.observations or {},
93
+ estimate_formula=self.estimate_formula,
94
+ )
95
+ if self.rule_id is not None:
96
+ result.update(rule_id=self.rule_id, rule_version=self.rule_version)
97
+ if self.estimate is not None:
98
+ result["estimate"] = deepcopy(self.estimate)
99
+ if self.ranking_requirements:
100
+ result["ranking_requirements"] = list(self.ranking_requirements)
101
+ return result
102
+
103
+ @classmethod
104
+ def from_dict(cls, data: dict[str, Any]) -> FindingCandidate:
105
+ values = {item.name: deepcopy(data[item.name]) for item in fields(cls) if item.name in data}
106
+ values["type"] = RecommendationType(values["type"])
107
+ return cls(**values)
108
+
109
+
110
+ @dataclass(frozen=True)
111
+ class AnalysisContext:
112
+ report: dict[str, Any]
113
+ graph: ExecutionGraph
114
+
115
+
116
+ @dataclass(frozen=True)
117
+ class FindingRule:
118
+ rule_id: str
119
+ version: str
120
+ detect: Callable[[AnalysisContext], list[FindingCandidate]]
121
+ ranking_requirements: tuple[str, ...] = ()
122
+
123
+ def __post_init__(self):
124
+ object.__setattr__(self, "ranking_requirements", requirements(self.ranking_requirements))
125
+
126
+
127
+ def run_rules(
128
+ context: AnalysisContext, rules: tuple[FindingRule, ...] | None = None
129
+ ) -> tuple[list[FindingCandidate], list[dict[str, str]]]:
130
+ """Run canonical local rules in declared order, retaining failure diagnostics."""
131
+ candidates: list[FindingCandidate] = []
132
+ errors = []
133
+ for rule in BUILTIN_RULES if rules is None else rules:
134
+ try:
135
+ detected = list(rule.detect(context))
136
+ if any(not isinstance(item, FindingCandidate) for item in detected):
137
+ raise TypeError("rule must return FindingCandidate objects")
138
+ for item in detected:
139
+ item.rule_id = rule.rule_id
140
+ item.rule_version = rule.version
141
+ item.ranking_requirements = rule.ranking_requirements
142
+ if rule in BUILTIN_RULES and item.estimate is None:
143
+ item.estimate = estimate_snapshot(item, context.report, context.graph.nodes)
144
+ candidates.extend(detected)
145
+ except Exception as exc:
146
+ _LOGGER.debug("Finding rule %s failed", rule.rule_id, exc_info=True)
147
+ errors.append(
148
+ {
149
+ "rule_id": rule.rule_id,
150
+ "rule_version": rule.version,
151
+ "error_type": _safe_error_type(exc),
152
+ "message": "Rule evaluation failed; see local debug logs.",
153
+ }
154
+ )
155
+ if not is_cost_evaluable(context.report.get("cost_status", "complete")):
156
+ for candidate in candidates:
157
+ candidate.estimated_cost_savings_usd = None
158
+ from agentloop.interoperability.evidence import qualify_candidates
159
+
160
+ qualify_candidates(context.report, candidates)
161
+ return candidates, errors
162
+
163
+
164
+ def _parallelization_cards(graph: ExecutionGraph) -> list[FindingCandidate]:
165
+ cards = []
166
+ for group in graph.parallelizable_groups():
167
+ cards.append(
168
+ FindingCandidate(
169
+ type=RecommendationType.PARALLELIZE_TOOLS,
170
+ title=f"Parallelize repeated `{group['name']}` tool calls",
171
+ why=f"{group['count']} repeated calls. {group['description']}",
172
+ rewrite_hint=PARALLELISM_REWRITE,
173
+ confidence=group["confidence"],
174
+ estimated_latency_savings_ms=group["estimated_savings_ms"],
175
+ affected_nodes=group["node_ids"],
176
+ evidence_level=group["evidence_level"],
177
+ assumptions=group["assumptions"],
178
+ observations=group["observations"],
179
+ estimate_formula=group["estimate_formula"],
180
+ )
181
+ )
182
+ return cards
183
+
184
+
185
+ def _context_cache_cards(report: dict[str, Any], graph: ExecutionGraph) -> list[FindingCandidate]:
186
+ ratio = report.get("repeated_context_ratio", 0.0)
187
+ if ratio is None:
188
+ return []
189
+ if ratio < 0.10:
190
+ return []
191
+ model_nodes = [node.node_id for node in graph.nodes if node.operation_kind == "model"]
192
+ if not model_nodes:
193
+ return []
194
+ current_cost = report.get("estimated_cost_usd", 0.0)
195
+ cost_evaluable = is_cost_evaluable(report.get("cost_status", "complete"))
196
+ return [
197
+ FindingCandidate(
198
+ type=RecommendationType.CACHE_CONTEXT,
199
+ title="Cache repeated prompt/context prefix",
200
+ why=f"Repeated context ratio is {ratio:.1%}, which suggests stable instructions or source text are being resent.",
201
+ rewrite_hint="Move stable instructions into cached prefixes, summaries, or framework-level memory instead of resending full text.",
202
+ confidence="high" if ratio >= 0.20 else "medium",
203
+ estimated_cost_savings_usd=(
204
+ current_cost
205
+ * min(ESTIMATORS["cache_context"].parameter("max_cost_fraction"), ratio)
206
+ if cost_evaluable
207
+ else None
208
+ ),
209
+ affected_nodes=model_nodes,
210
+ )
211
+ ]
212
+
213
+
214
+ def _retry_cards(graph: ExecutionGraph) -> list[FindingCandidate]:
215
+ retry_nodes = [node for node in graph.nodes if node.operation_kind == "retry"]
216
+ if not retry_nodes:
217
+ return []
218
+ retry_time = sum(node.duration_ms for node in retry_nodes)
219
+ return [
220
+ FindingCandidate(
221
+ type=RecommendationType.ADD_SCHEMA_VALIDATION,
222
+ title="Reduce retry loop with structured outputs",
223
+ why=f"Observed {len(retry_nodes)} retry span(s), costing {retry_time / 1000:.2f}s.",
224
+ rewrite_hint="Add schema validation, JSON mode, constrained decoding, or a cheap repair prompt before rerunning a full step.",
225
+ confidence="high",
226
+ estimated_latency_savings_ms=retry_time
227
+ * ESTIMATORS["add_schema_validation"].parameter("latency_fraction"),
228
+ affected_nodes=[node.node_id for node in retry_nodes],
229
+ )
230
+ ]
231
+
232
+
233
+ def _batch_model_cards(graph: ExecutionGraph) -> list[FindingCandidate]:
234
+ cards = []
235
+ for name, nodes in model_batch_groups(graph):
236
+ duration = sum(node.duration_ms for node in nodes)
237
+ cards.append(
238
+ FindingCandidate(
239
+ type=RecommendationType.BATCH_MODEL_CALLS,
240
+ title=f"Batch repeated `{name}` model calls",
241
+ why=f"{len(nodes)} `{name}` calls have the same role and may be batchable.",
242
+ rewrite_hint="Batch documents/items into one prompt or use map-reduce only when outputs truly need independent reasoning.",
243
+ confidence="medium",
244
+ estimated_latency_savings_ms=duration
245
+ * ESTIMATORS["batch_model_calls"].parameter("latency_fraction"),
246
+ affected_nodes=[node.node_id for node in nodes],
247
+ )
248
+ )
249
+ return cards
250
+
251
+
252
+ def _routing_cards(graph: ExecutionGraph) -> list[FindingCandidate]:
253
+ cards = []
254
+ for node in graph.nodes:
255
+ if node.operation_kind != "model" or not node.model:
256
+ continue
257
+ if node.token_provenance == UNAVAILABLE or (
258
+ isinstance(node.metadata, Mapping) and node.metadata.get("usage_available") is False
259
+ ):
260
+ continue
261
+ if "mini" in node.model.lower():
262
+ continue
263
+ if node.output_tokens <= 400 and node.input_tokens <= 2500:
264
+ cards.append(
265
+ FindingCandidate(
266
+ type=RecommendationType.ROUTE_TO_SMALLER_MODEL,
267
+ title=f"Route `{node.name}` to a cheaper model",
268
+ why=f"`{node.name}` is a relatively small model step using {node.total_tokens} tokens on {node.model}.",
269
+ rewrite_hint="Try a smaller model for planning, summarization, extraction, or verification steps and keep the larger model for final synthesis.",
270
+ confidence="low",
271
+ estimated_latency_savings_ms=node.duration_ms
272
+ * ESTIMATORS["route_to_smaller_model"].parameter("latency_fraction"),
273
+ affected_nodes=[node.node_id],
274
+ )
275
+ )
276
+ return cards[:3]
277
+
278
+
279
+ def _split_large_step_cards(graph: ExecutionGraph) -> list[FindingCandidate]:
280
+ cards = []
281
+ for node in graph.nodes:
282
+ if node.operation_kind == "model" and node.total_tokens >= 4000:
283
+ cards.append(
284
+ FindingCandidate(
285
+ type=RecommendationType.SPLIT_LARGE_STEP,
286
+ title=f"Split or compress `{node.name}`",
287
+ why=f"`{node.name}` used {node.total_tokens} tokens, making it a large and fragile step.",
288
+ rewrite_hint="Split into retrieve-filter-summarize or compress the context before the final reasoning call.",
289
+ confidence="medium",
290
+ estimated_latency_savings_ms=node.duration_ms
291
+ * ESTIMATORS["split_large_step"].parameter("latency_fraction"),
292
+ affected_nodes=[node.node_id],
293
+ )
294
+ )
295
+ return cards
296
+
297
+
298
+ def _runaway_loop_cards(graph: ExecutionGraph) -> list[FindingCandidate]:
299
+ if graph.execution is not None:
300
+ return [] # Repeated pipeline stages do not establish an agent loop.
301
+ cards = []
302
+ groups: dict[tuple[str, str], list[Any]] = {}
303
+ for node in graph.nodes:
304
+ groups.setdefault((node.operation_kind, node.name), []).append(node)
305
+
306
+ for (event_type, name), nodes in groups.items():
307
+ if len(nodes) < 8:
308
+ continue
309
+ duration = sum(node.duration_ms for node in nodes)
310
+ cards.append(
311
+ FindingCandidate(
312
+ type=RecommendationType.RUNAWAY_LOOP,
313
+ title=f"Add a loop guardrail around `{name}`",
314
+ why=f"`{name}` ran {len(nodes)} times in one trace, which suggests an unbounded or weakly bounded agent loop.",
315
+ rewrite_hint="Add max-iteration, max-cost, and unchanged-state guards before the workflow can keep looping.",
316
+ confidence="high" if len(nodes) >= 12 else "medium",
317
+ estimated_latency_savings_ms=duration
318
+ * ESTIMATORS["runaway_loop"].parameter("latency_fraction"),
319
+ affected_nodes=[node.node_id for node in nodes],
320
+ )
321
+ )
322
+ return cards[:3]
323
+
324
+
325
+ def _tool_oscillation_cards(graph: ExecutionGraph) -> list[FindingCandidate]:
326
+ if graph.execution is not None:
327
+ return [] # Generic workflow ordering is not evidence of agent oscillation.
328
+ tool_nodes = [node for node in graph.nodes if node.operation_kind == "tool"]
329
+ if len(tool_nodes) < 4:
330
+ return []
331
+
332
+ cards = []
333
+ index = 0
334
+ while index <= len(tool_nodes) - 4:
335
+ first = tool_nodes[index].name
336
+ second = tool_nodes[index + 1].name
337
+ if first == second:
338
+ index += 1
339
+ continue
340
+
341
+ window = tool_nodes[index : index + 4]
342
+ if [node.name for node in window] != [first, second, first, second]:
343
+ index += 1
344
+ continue
345
+
346
+ end = index + 4
347
+ while end < len(tool_nodes) and tool_nodes[end].name == (
348
+ first if (end - index) % 2 == 0 else second
349
+ ):
350
+ end += 1
351
+ oscillating = tool_nodes[index:end]
352
+ duration = sum(node.duration_ms for node in oscillating)
353
+ cards.append(
354
+ FindingCandidate(
355
+ type=RecommendationType.TOOL_OSCILLATION,
356
+ title=f"Stop `{first}`/`{second}` tool oscillation",
357
+ why=f"Observed {len(oscillating)} alternating `{first}` and `{second}` tool calls.",
358
+ rewrite_hint="Add a state-change check or decision memo so the agent does not repeat equivalent tool transitions.",
359
+ confidence="medium",
360
+ estimated_latency_savings_ms=duration
361
+ * ESTIMATORS["tool_oscillation"].parameter("latency_fraction"),
362
+ affected_nodes=[node.node_id for node in oscillating],
363
+ )
364
+ )
365
+ index = end
366
+
367
+ return sorted(cards, key=lambda card: card.estimated_latency_savings_ms, reverse=True)[:3]
368
+
369
+
370
+ def _semantic_cards(context, family):
371
+ if not context.report.get("semantic_waste"):
372
+ return []
373
+ from agentloop.semantic_rules import semantic_candidates
374
+
375
+ return semantic_candidates(context, family)
376
+
377
+
378
+ BUILTIN_RULES = (
379
+ FindingRule("parallelize_tools", "1.1", lambda ctx: _parallelization_cards(ctx.graph)),
380
+ FindingRule("cache_context", "1.1", lambda ctx: _context_cache_cards(ctx.report, ctx.graph)),
381
+ FindingRule("add_schema_validation", "1.1", lambda ctx: _retry_cards(ctx.graph)),
382
+ FindingRule("batch_model_calls", "1.2", lambda ctx: _batch_model_cards(ctx.graph)),
383
+ FindingRule("route_to_smaller_model", "1.2", lambda ctx: _routing_cards(ctx.graph)),
384
+ FindingRule("split_large_step", "1.1", lambda ctx: _split_large_step_cards(ctx.graph)),
385
+ FindingRule("runaway_loop", "1.1", lambda ctx: _runaway_loop_cards(ctx.graph)),
386
+ FindingRule("tool_oscillation", "1.1", lambda ctx: _tool_oscillation_cards(ctx.graph)),
387
+ FindingRule(
388
+ "semantic_redundancy", "1.0", lambda ctx: _semantic_cards(ctx, "semantic_redundancy")
389
+ ),
390
+ FindingRule("low_contribution", "1.0", lambda ctx: _semantic_cards(ctx, "low_contribution")),
391
+ FindingRule(
392
+ "semantic_no_progress", "1.0", lambda ctx: _semantic_cards(ctx, "semantic_no_progress")
393
+ ),
394
+ FindingRule("retry_usefulness", "1.0", lambda ctx: _semantic_cards(ctx, "retry_usefulness")),
395
+ FindingRule("context_relevance", "1.0", lambda ctx: _semantic_cards(ctx, "context_relevance")),
396
+ )
agentloop/runtime.py ADDED
@@ -0,0 +1,286 @@
1
+ from __future__ import annotations
2
+
3
+ import os
4
+ from dataclasses import dataclass
5
+ from pathlib import Path
6
+ from typing import TYPE_CHECKING, Any
7
+
8
+ if TYPE_CHECKING:
9
+ from agentloop.retention import RetentionSession
10
+
11
+ _TRUE_VALUES = {"1", "true", "yes", "on"}
12
+
13
+
14
+ class _ClearSentinel:
15
+ """Sentinel passed to :func:`init` to explicitly clear an optional value.
16
+
17
+ ``init(api_key=None)`` means "keep the current value" (so environment and
18
+ prior configuration survive a reconfigure). Passing ``CLEAR`` instead sets
19
+ the value back to ``None``.
20
+ """
21
+
22
+ def __repr__(self) -> str:
23
+ return "CLEAR"
24
+
25
+
26
+ CLEAR = _ClearSentinel()
27
+
28
+
29
+ class FinalizationError(RuntimeError):
30
+ """Raised by :func:`finalize_trace` when a destination fails and
31
+ ``fail_silently`` is ``False``.
32
+
33
+ ``result`` holds the partial finalization result for destinations that
34
+ completed before the failure, so already-completed work is not lost.
35
+ ``errors`` lists the destination-tagged failures.
36
+ """
37
+
38
+ def __init__(
39
+ self,
40
+ message: str,
41
+ *,
42
+ result: dict[str, Any],
43
+ errors: list[dict[str, str]],
44
+ ) -> None:
45
+ super().__init__(message)
46
+ self.result = result
47
+ self.errors = errors
48
+
49
+
50
+ @dataclass
51
+ class AgentLoopRuntimeConfig:
52
+ api_url: str = "http://127.0.0.1:8000"
53
+ api_key: str | None = None
54
+ project_id: str = "default"
55
+ auto_upload: bool = False
56
+ auto_store: bool = False
57
+ fail_silently: bool = True
58
+ export_dir: Path | None = None
59
+ retention: RetentionSession | None = None
60
+
61
+
62
+ _runtime = AgentLoopRuntimeConfig()
63
+ _last_error: str | None = None
64
+
65
+
66
+ def _env_bool(name: str, default: bool = False) -> bool:
67
+ value = os.getenv(name)
68
+ if value is None:
69
+ return default
70
+ return value.lower() in _TRUE_VALUES
71
+
72
+
73
+ def init(
74
+ *,
75
+ api_url: str | None = None,
76
+ api_key: str | _ClearSentinel | None = None,
77
+ project_id: str | None = None,
78
+ auto_upload: bool | None = None,
79
+ auto_store: bool | None = None,
80
+ fail_silently: bool | None = None,
81
+ export_dir: str | Path | _ClearSentinel | None = None,
82
+ retention: RetentionSession | _ClearSentinel | None = None,
83
+ ) -> AgentLoopRuntimeConfig:
84
+ """Configure AgentLoop once at process startup.
85
+
86
+ Examples:
87
+
88
+ ```python
89
+ import agentloop
90
+
91
+ agentloop.init(api_key="al_xxx", project_id="demo", auto_upload=True)
92
+ ```
93
+
94
+ Environment variables are also supported:
95
+ - `AGENTLOOP_API_URL`
96
+ - `AGENTLOOP_API_KEY`
97
+ - `AGENTLOOP_PROJECT_ID`
98
+ - `AGENTLOOP_AUTO_UPLOAD`
99
+ - `AGENTLOOP_AUTO_STORE`
100
+ - `AGENTLOOP_FAIL_SILENTLY`
101
+ - `AGENTLOOP_EXPORT_DIR`
102
+
103
+ Optional values (`api_key`, `export_dir`) treat ``None`` as "keep the current
104
+ value". Pass :data:`CLEAR` to reset one of them back to ``None``.
105
+ """
106
+
107
+ global _runtime
108
+
109
+ from agentloop.retention import RetentionSession
110
+
111
+ if (
112
+ retention is not None
113
+ and retention is not CLEAR
114
+ and not isinstance(retention, RetentionSession)
115
+ ):
116
+ raise TypeError("retention must be RetentionSession, CLEAR or None")
117
+
118
+ if api_key is CLEAR:
119
+ resolved_api_key: str | None = None
120
+ elif api_key is not None:
121
+ resolved_api_key = api_key # type: ignore[assignment]
122
+ else:
123
+ resolved_api_key = os.getenv("AGENTLOOP_API_KEY", _runtime.api_key)
124
+
125
+ if export_dir is CLEAR:
126
+ resolved_export_dir: Path | None = None
127
+ elif export_dir is not None:
128
+ resolved_export_dir = Path(export_dir) # type: ignore[arg-type]
129
+ else:
130
+ env_export_dir = os.getenv("AGENTLOOP_EXPORT_DIR")
131
+ resolved_export_dir = Path(env_export_dir) if env_export_dir else _runtime.export_dir
132
+
133
+ _runtime = AgentLoopRuntimeConfig(
134
+ api_url=api_url or os.getenv("AGENTLOOP_API_URL", _runtime.api_url),
135
+ api_key=resolved_api_key,
136
+ project_id=project_id or os.getenv("AGENTLOOP_PROJECT_ID", _runtime.project_id),
137
+ auto_upload=auto_upload
138
+ if auto_upload is not None
139
+ else _env_bool("AGENTLOOP_AUTO_UPLOAD", _runtime.auto_upload),
140
+ auto_store=auto_store
141
+ if auto_store is not None
142
+ else _env_bool("AGENTLOOP_AUTO_STORE", _runtime.auto_store),
143
+ fail_silently=fail_silently
144
+ if fail_silently is not None
145
+ else _env_bool("AGENTLOOP_FAIL_SILENTLY", _runtime.fail_silently),
146
+ export_dir=resolved_export_dir,
147
+ retention=None
148
+ if retention is CLEAR
149
+ else retention
150
+ if retention is not None
151
+ else _runtime.retention,
152
+ )
153
+ return _runtime
154
+
155
+
156
+ def configure_from_env() -> AgentLoopRuntimeConfig:
157
+ return init()
158
+
159
+
160
+ def get_runtime_config() -> AgentLoopRuntimeConfig:
161
+ return _runtime
162
+
163
+
164
+ def get_last_error() -> str | None:
165
+ return _last_error
166
+
167
+
168
+ def reset_runtime() -> None:
169
+ global _runtime, _last_error
170
+ _runtime = AgentLoopRuntimeConfig()
171
+ _last_error = None
172
+
173
+
174
+ def should_auto_export() -> bool:
175
+ return bool(_runtime.auto_upload or _runtime.auto_store or _runtime.export_dir)
176
+
177
+
178
+ def finalize_trace(trace: Any) -> dict[str, Any]:
179
+ """Apply configured end-of-run side effects.
180
+
181
+ Depending on runtime config, this exports a JSON file, saves to the local
182
+ persistent store, and/or uploads to the configured API. Each destination runs
183
+ in its own error boundary in a deterministic order (export, store, upload), so
184
+ a failure in one cannot prevent the others from running.
185
+
186
+ When ``fail_silently`` is ``True`` (the default), every configured destination
187
+ is attempted and failures are collected in ``result["errors"]``, each tagged
188
+ with its ``destination``. When ``fail_silently`` is ``False``, the first failing
189
+ destination raises :class:`FinalizationError` (fail-fast); its ``result``
190
+ attribute still carries the destinations that completed before the failure.
191
+
192
+ A finalization in which every attempted destination succeeds clears any stale
193
+ process-global error so :func:`get_last_error` reflects the latest outcome.
194
+ """
195
+
196
+ global _last_error
197
+ result: dict[str, Any] = {
198
+ "exported_path": None,
199
+ "stored": False,
200
+ "uploaded": False,
201
+ "errors": [],
202
+ }
203
+ if _runtime.retention is not None:
204
+ try:
205
+ trace = _runtime.retention.retain(trace)
206
+ result["retention"] = {
207
+ "retained": trace is not None,
208
+ "sampling": _runtime.retention.summary(),
209
+ }
210
+ except Exception as exc:
211
+ # Never fall back to raw export after a privacy/retention failure.
212
+ errors = [
213
+ {
214
+ "destination": "retention",
215
+ "error": "Retention failed; no destinations attempted.",
216
+ }
217
+ ]
218
+ result["errors"] = errors
219
+ _last_error = errors[0]["error"]
220
+ if not _runtime.fail_silently:
221
+ raise FinalizationError(_last_error, result=result, errors=errors) from exc
222
+ return result
223
+ if trace is None:
224
+ _last_error = None
225
+ return result
226
+
227
+ def _do_export() -> None:
228
+ export_dir = _runtime.export_dir
229
+ if export_dir is None: # pragma: no cover - guarded before registration
230
+ return
231
+ export_dir.mkdir(parents=True, exist_ok=True)
232
+ out = export_dir / f"{trace.run_id}.json"
233
+ trace.export_json(out)
234
+ result["exported_path"] = str(out)
235
+
236
+ def _do_store() -> None:
237
+ from agentloop.store import get_store
238
+
239
+ db = get_store()
240
+ db.init()
241
+ db.save_trace(trace, project_id=_runtime.project_id)
242
+ result["stored"] = True
243
+
244
+ def _do_upload() -> None:
245
+ from agentloop.client import AgentLoopClient
246
+
247
+ client = AgentLoopClient(base_url=_runtime.api_url, api_key=_runtime.api_key)
248
+ result["upload_response"] = client.upload_trace(trace)
249
+ result["uploaded"] = True
250
+
251
+ destinations: list[tuple[str, Any]] = []
252
+ if _runtime.export_dir is not None:
253
+ destinations.append(("export", _do_export))
254
+ if _runtime.auto_store:
255
+ destinations.append(("store", _do_store))
256
+ if _runtime.auto_upload:
257
+ destinations.append(("upload", _do_upload))
258
+
259
+ errors: list[dict[str, str]] = []
260
+ for destination, action in destinations:
261
+ try:
262
+ action()
263
+ except Exception as exc:
264
+ errors.append({"destination": destination, "error": str(exc)})
265
+ if not _runtime.fail_silently:
266
+ result["errors"] = errors
267
+ _last_error = f"{destination}: {exc}"
268
+ raise FinalizationError(
269
+ f"{destination} finalization failed: {exc}",
270
+ result=result,
271
+ errors=errors,
272
+ ) from exc
273
+
274
+ result["errors"] = errors
275
+ if errors:
276
+ last = errors[-1]
277
+ _last_error = f"{last['destination']}: {last['error']}"
278
+ elif destinations:
279
+ # Every attempted destination succeeded; drop any stale error state.
280
+ _last_error = None
281
+
282
+ return result
283
+
284
+
285
+ # Load env-driven defaults on import without forcing users to call init().
286
+ configure_from_env()