loopgain 0.4.2__tar.gz → 0.5.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. {loopgain-0.4.2 → loopgain-0.5.0}/PKG-INFO +3 -11
  2. {loopgain-0.4.2 → loopgain-0.5.0}/README.md +2 -10
  3. {loopgain-0.4.2 → loopgain-0.5.0}/loopgain/_version.py +1 -1
  4. {loopgain-0.4.2 → loopgain-0.5.0}/loopgain/core.py +16 -77
  5. {loopgain-0.4.2 → loopgain-0.5.0}/loopgain/telemetry.py +75 -40
  6. {loopgain-0.4.2 → loopgain-0.5.0}/loopgain.egg-info/PKG-INFO +3 -11
  7. {loopgain-0.4.2 → loopgain-0.5.0}/tests/test_core.py +1 -121
  8. {loopgain-0.4.2 → loopgain-0.5.0}/tests/test_stress.py +1 -15
  9. {loopgain-0.4.2 → loopgain-0.5.0}/tests/test_telemetry.py +128 -41
  10. {loopgain-0.4.2 → loopgain-0.5.0}/LICENSE +0 -0
  11. {loopgain-0.4.2 → loopgain-0.5.0}/loopgain/__init__.py +0 -0
  12. {loopgain-0.4.2 → loopgain-0.5.0}/loopgain/__main__.py +0 -0
  13. {loopgain-0.4.2 → loopgain-0.5.0}/loopgain/classifier.py +0 -0
  14. {loopgain-0.4.2 → loopgain-0.5.0}/loopgain/cli.py +0 -0
  15. {loopgain-0.4.2 → loopgain-0.5.0}/loopgain/funnel.py +0 -0
  16. {loopgain-0.4.2 → loopgain-0.5.0}/loopgain/integrations/__init__.py +0 -0
  17. {loopgain-0.4.2 → loopgain-0.5.0}/loopgain/integrations/autogen.py +0 -0
  18. {loopgain-0.4.2 → loopgain-0.5.0}/loopgain/integrations/claude_agent_sdk.py +0 -0
  19. {loopgain-0.4.2 → loopgain-0.5.0}/loopgain/integrations/crewai.py +0 -0
  20. {loopgain-0.4.2 → loopgain-0.5.0}/loopgain/integrations/langchain.py +0 -0
  21. {loopgain-0.4.2 → loopgain-0.5.0}/loopgain/integrations/langgraph.py +0 -0
  22. {loopgain-0.4.2 → loopgain-0.5.0}/loopgain/integrations/openai_agents.py +0 -0
  23. {loopgain-0.4.2 → loopgain-0.5.0}/loopgain.egg-info/SOURCES.txt +0 -0
  24. {loopgain-0.4.2 → loopgain-0.5.0}/loopgain.egg-info/dependency_links.txt +0 -0
  25. {loopgain-0.4.2 → loopgain-0.5.0}/loopgain.egg-info/entry_points.txt +0 -0
  26. {loopgain-0.4.2 → loopgain-0.5.0}/loopgain.egg-info/requires.txt +0 -0
  27. {loopgain-0.4.2 → loopgain-0.5.0}/loopgain.egg-info/top_level.txt +0 -0
  28. {loopgain-0.4.2 → loopgain-0.5.0}/pyproject.toml +0 -0
  29. {loopgain-0.4.2 → loopgain-0.5.0}/setup.cfg +0 -0
  30. {loopgain-0.4.2 → loopgain-0.5.0}/tests/test_classifier_mock_validation.py +0 -0
  31. {loopgain-0.4.2 → loopgain-0.5.0}/tests/test_classifier_synthetic.py +0 -0
  32. {loopgain-0.4.2 → loopgain-0.5.0}/tests/test_funnel.py +0 -0
  33. {loopgain-0.4.2 → loopgain-0.5.0}/tests/test_integrations.py +0 -0
  34. {loopgain-0.4.2 → loopgain-0.5.0}/tests/test_termination_safety.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: loopgain
3
- Version: 0.4.2
3
+ Version: 0.5.0
4
4
  Summary: An open-source cost controller for AI agent loops. Stops a loop when it has actually converged and rolls back before it degrades — replacing the max_iterations guess with a real-time loop-gain (Aβ) monitor with five named threshold bands and best-so-far rollback.
5
5
  Author-email: Dave Fitzsimmons <hello@loopgain.ai>
6
6
  License: Apache-2.0
@@ -102,7 +102,6 @@ result = lg.result
102
102
  print(result.outcome) # "converged" | "oscillating" | "diverged" | "stalled" | "max_iterations"
103
103
  print(result.best_output) # the lowest-error iteration's output
104
104
  print(result.iterations_used)
105
- print(result.gain_margin) # 1 / max(Aβ_smooth)
106
105
  print(result.savings_vs_fixed_cap)
107
106
  ```
108
107
 
@@ -183,6 +182,7 @@ LoopGain saves money by stopping a loop once it stops improving — fewer iterat
183
182
 
184
183
  - **Savings depend on your workload.** Loops that usually succeed fast save the most (~96%); adversarial, failure-prone loops save less (~78–84%). The headline is a blend — run the benchmark on your own loops before quoting a number.
185
184
  - **LoopGain detects convergence, not correctness.** It stops when your error signal stops improving — which means more iterations won't help, *not* that the loop succeeded. On the benchmark this preserved quality (it rarely stopped early on a worse output; false-stop rate ≤4.5%), but a loop can stall with the error still above zero — a plateau at, say, 2 failing tests. So check `result.best_error` (or your own pass/fail) before you trust the output: if it plateaued short of your target, that's a quality gap LoopGain can't see, and a false stop that forces a rerun is the one way it eats into the savings. LoopGain decides *when to stop*; you decide *whether the answer is good enough*.
185
+ - **LoopGain is only as right as your verifier.** It acts on the error signal you give it. If your verifier reports zero errors, LoopGain trusts that and stops — so a verifier with blind spots can report success on an answer that is still wrong, and LoopGain will confidently stop there. This is not the plateau case above: the error reads zero and the loop looks like a clean success, so neither LoopGain nor its convergence signal can flag it. The quality of the stop is bounded by the quality of the check behind your error signal. Pair LoopGain with the strongest verifier you can afford at the stop — executable tests over a sampled subset, a schema or type check over a vibe, a held-out check the loop didn't optimize against.
186
186
 
187
187
  ---
188
188
 
@@ -212,17 +212,9 @@ Returns `False` once a terminal state fires.
212
212
 
213
213
  Current state name. One of `INIT`, `FAST_CONVERGE`, `CONVERGING`, `STALLING`, `OSCILLATING`, `DIVERGING`, `TARGET_MET`, `MAX_ITERATIONS`. The corresponding terminal `result.outcome` values are `converged`, `oscillating`, `diverged`, `stalled` (v0.2 trajectory mode only — STALLING terminating after 2 consecutive readings), `max_iterations`, or `in_progress`.
214
214
 
215
- ### `lg.eta -> int | None`
216
-
217
- Best-effort closed-form estimate of iterations remaining, exposed for instrumentation. Returns `None` whenever it isn't well-defined — which is most of the time on real, jump-dominated loops, so don't depend on it for control.
218
-
219
- ### `lg.gain_margin -> float | None`
220
-
221
- `1 / max(Aβ_smooth)`. `> 1` means stable headroom across the entire run.
222
-
223
215
  ### `lg.result -> LoopGainResult`
224
216
 
225
- Terminal result with `outcome`, `iterations_used`, `best_index`, `best_output`, `best_error`, `convergence_profile`, `error_history`, `gain_margin`, `savings_vs_fixed_cap`. Safe to call mid-loop.
217
+ Terminal result with `outcome`, `iterations_used`, `best_index`, `best_output`, `best_error`, `convergence_profile`, `error_history`, `savings_vs_fixed_cap`. Safe to call mid-loop.
226
218
 
227
219
  ### `lg.send_telemetry(endpoint, token, workload_id=None, timeout=2.0, allow_insecure=False, framework=None, loop_type=None, team=None, include_per_iteration=True) -> bool`
228
220
 
@@ -53,7 +53,6 @@ result = lg.result
53
53
  print(result.outcome) # "converged" | "oscillating" | "diverged" | "stalled" | "max_iterations"
54
54
  print(result.best_output) # the lowest-error iteration's output
55
55
  print(result.iterations_used)
56
- print(result.gain_margin) # 1 / max(Aβ_smooth)
57
56
  print(result.savings_vs_fixed_cap)
58
57
  ```
59
58
 
@@ -134,6 +133,7 @@ LoopGain saves money by stopping a loop once it stops improving — fewer iterat
134
133
 
135
134
  - **Savings depend on your workload.** Loops that usually succeed fast save the most (~96%); adversarial, failure-prone loops save less (~78–84%). The headline is a blend — run the benchmark on your own loops before quoting a number.
136
135
  - **LoopGain detects convergence, not correctness.** It stops when your error signal stops improving — which means more iterations won't help, *not* that the loop succeeded. On the benchmark this preserved quality (it rarely stopped early on a worse output; false-stop rate ≤4.5%), but a loop can stall with the error still above zero — a plateau at, say, 2 failing tests. So check `result.best_error` (or your own pass/fail) before you trust the output: if it plateaued short of your target, that's a quality gap LoopGain can't see, and a false stop that forces a rerun is the one way it eats into the savings. LoopGain decides *when to stop*; you decide *whether the answer is good enough*.
136
+ - **LoopGain is only as right as your verifier.** It acts on the error signal you give it. If your verifier reports zero errors, LoopGain trusts that and stops — so a verifier with blind spots can report success on an answer that is still wrong, and LoopGain will confidently stop there. This is not the plateau case above: the error reads zero and the loop looks like a clean success, so neither LoopGain nor its convergence signal can flag it. The quality of the stop is bounded by the quality of the check behind your error signal. Pair LoopGain with the strongest verifier you can afford at the stop — executable tests over a sampled subset, a schema or type check over a vibe, a held-out check the loop didn't optimize against.
137
137
 
138
138
  ---
139
139
 
@@ -163,17 +163,9 @@ Returns `False` once a terminal state fires.
163
163
 
164
164
  Current state name. One of `INIT`, `FAST_CONVERGE`, `CONVERGING`, `STALLING`, `OSCILLATING`, `DIVERGING`, `TARGET_MET`, `MAX_ITERATIONS`. The corresponding terminal `result.outcome` values are `converged`, `oscillating`, `diverged`, `stalled` (v0.2 trajectory mode only — STALLING terminating after 2 consecutive readings), `max_iterations`, or `in_progress`.
165
165
 
166
- ### `lg.eta -> int | None`
167
-
168
- Best-effort closed-form estimate of iterations remaining, exposed for instrumentation. Returns `None` whenever it isn't well-defined — which is most of the time on real, jump-dominated loops, so don't depend on it for control.
169
-
170
- ### `lg.gain_margin -> float | None`
171
-
172
- `1 / max(Aβ_smooth)`. `> 1` means stable headroom across the entire run.
173
-
174
166
  ### `lg.result -> LoopGainResult`
175
167
 
176
- Terminal result with `outcome`, `iterations_used`, `best_index`, `best_output`, `best_error`, `convergence_profile`, `error_history`, `gain_margin`, `savings_vs_fixed_cap`. Safe to call mid-loop.
168
+ Terminal result with `outcome`, `iterations_used`, `best_index`, `best_output`, `best_error`, `convergence_profile`, `error_history`, `savings_vs_fixed_cap`. Safe to call mid-loop.
177
169
 
178
170
  ### `lg.send_telemetry(endpoint, token, workload_id=None, timeout=2.0, allow_insecure=False, framework=None, loop_type=None, team=None, include_per_iteration=True) -> bool`
179
171
 
@@ -7,4 +7,4 @@ from here so the value never drifts between ``__version__`` and the
7
7
  ``pyproject.toml``) for each release.
8
8
  """
9
9
 
10
- __version__ = "0.4.2"
10
+ __version__ = "0.5.0"
@@ -9,8 +9,7 @@ real-time loop-gain monitor that classifies the loop into one of five
9
9
  named states and decides whether to continue, stop, or roll back.
10
10
 
11
11
  The math is foundational EE control theory. The product layer is the
12
- threshold bands, the best-so-far buffer, the ETA prediction, and the
13
- clean Python API.
12
+ threshold bands, the best-so-far buffer, and the clean Python API.
14
13
 
15
14
  License: Apache-2.0
16
15
  """
@@ -121,28 +120,10 @@ class LoopGainResult:
121
120
  error_history: list[float] = field(default_factory=list)
122
121
  """All observed error magnitudes, in order."""
123
122
 
124
- gain_margin: Optional[float] = None
125
- """``1 / max(Aβ_smooth)``. > 1 means stable headroom; < 1 means the
126
- loop crossed into oscillation/divergence at some point."""
127
-
128
123
  savings_vs_fixed_cap: Optional[int] = None
129
124
  """Iterations saved versus the assumed fixed cap (default 10).
130
125
  Zero if the loop hit ``max_iterations``; otherwise non-negative."""
131
126
 
132
- first_eta_prediction: Optional[int] = None
133
- """First non-None ``eta`` snapshot captured during the loop —
134
- the predicted iterations-remaining at the moment the prediction
135
- became computable. ``None`` if no prediction was ever made
136
- (e.g., ``target_error`` is ``None`` or ``0`` so the prediction
137
- formula is undefined, loop never converged toward target, or the
138
- loop terminated before two observations)."""
139
-
140
- first_eta_at_iteration: Optional[int] = None
141
- """Iteration count when ``first_eta_prediction`` was captured.
142
- ``None`` if no prediction was ever made. Predicted *total*
143
- iterations = ``first_eta_at_iteration + first_eta_prediction``,
144
- comparable to ``iterations_used`` for calibration."""
145
-
146
127
 
147
128
  class LoopGain:
148
129
  """Barkhausen stability monitor for AI agent loops.
@@ -239,8 +220,6 @@ class LoopGain:
239
220
  self._state: str = INIT
240
221
  self._state_history: list[str] = []
241
222
  self._terminal: bool = False
242
- self._first_eta_prediction: Optional[int] = None
243
- self._first_eta_at_iteration: Optional[int] = None
244
223
 
245
224
  # Opt-in anonymous funnel telemetry (see loopgain.funnel). No-op
246
225
  # unless the user has explicitly opted in; fully fail-silent and
@@ -348,17 +327,6 @@ class LoopGain:
348
327
  self._state = MAX_ITERATIONS
349
328
  self._terminal = True
350
329
 
351
- # Snapshot the first computable eta prediction for calibration.
352
- # eta is None until smoothing settles and the loop looks convergent;
353
- # we capture the *first* value it produces and the iteration it was
354
- # produced at, so predicted_total = at_iter + eta is comparable to
355
- # iterations_used.
356
- if self._first_eta_prediction is None:
357
- eta_now = self.eta
358
- if eta_now is not None and eta_now > 0:
359
- self._first_eta_prediction = eta_now
360
- self._first_eta_at_iteration = len(self._error_history)
361
-
362
330
  # Funnel telemetry: if this observation drove the loop terminal
363
331
  # (oscillating / diverging / stalled / max-iterations), count the
364
332
  # coarse outcome. The TARGET_MET case is handled at its early return.
@@ -382,46 +350,6 @@ class LoopGain:
382
350
  """Current state name."""
383
351
  return self._state
384
352
 
385
- @property
386
- def eta(self) -> Optional[int]:
387
- """Predicted iterations remaining to reach ``target_error``.
388
-
389
- Closed-form Barkhausen prediction:
390
-
391
- n_remaining = log(E_target / E_current) / log(Aβ_smooth)
392
-
393
- Returns ``None`` when the prediction isn't well-defined:
394
- no Aβ yet, ``target_error`` is ``None`` (no target to predict
395
- against) or ``0`` (the formula has a log-of-zero singularity),
396
- target already met, or ``Aβ_smooth >= 1`` (non-converging gain).
397
- """
398
- if not self._smoothed_history or not self._error_history:
399
- return None
400
- if self.target_error is None or self.target_error <= 0:
401
- return None
402
- e_current = self._error_history[-1]
403
- if e_current <= self.target_error:
404
- return 0
405
- ab_smooth = self._smoothed_history[-1]
406
- if ab_smooth >= 1.0 or ab_smooth <= 0:
407
- return None
408
- n = math.log(self.target_error / e_current) / math.log(ab_smooth)
409
- return max(0, math.ceil(n))
410
-
411
- @property
412
- def gain_margin(self) -> Optional[float]:
413
- """Gain margin ``GM = 1 / max(Aβ_smooth)``.
414
-
415
- ``GM > 1`` means the loop never crossed into oscillation. The
416
- larger, the more headroom. Returns ``None`` if no Aβ data yet.
417
- """
418
- if not self._smoothed_history:
419
- return None
420
- max_g = max(self._smoothed_history)
421
- if max_g == 0:
422
- return float("inf")
423
- return 1.0 / max_g
424
-
425
353
  @property
426
354
  def result(self) -> LoopGainResult:
427
355
  """Construct the terminal result. Safe to call any time."""
@@ -468,10 +396,7 @@ class LoopGain:
468
396
  best_error=best_error,
469
397
  convergence_profile=list(self._smoothed_history),
470
398
  error_history=list(self._error_history),
471
- gain_margin=self.gain_margin,
472
399
  savings_vs_fixed_cap=savings,
473
- first_eta_prediction=self._first_eta_prediction,
474
- first_eta_at_iteration=self._first_eta_at_iteration,
475
400
  )
476
401
 
477
402
  # ----- Internal helpers -----
@@ -514,6 +439,8 @@ class LoopGain:
514
439
  loop_type: Optional[str] = None,
515
440
  team: Optional[str] = None,
516
441
  include_per_iteration: bool = True,
442
+ retries: int = 2,
443
+ retry_backoff: float = 0.25,
517
444
  ) -> bool:
518
445
  """Send anonymized telemetry to a receiver endpoint.
519
446
 
@@ -544,6 +471,12 @@ class LoopGain:
544
471
  per-iteration Aβ + error trajectories (capped) so the
545
472
  dashboard's Loop Detail scrubber works. Set ``False`` to
546
473
  send only aggregate summary stats.
474
+ retries: Additional attempts if a send fails *transiently*
475
+ (timeout, connection error, 5xx/429). Default 2 (up to 3
476
+ attempts). Set to 0 for single-shot. Deterministic failures
477
+ (bad token, etc.) are never retried.
478
+ retry_backoff: Base seconds between attempts; the nth retry waits
479
+ ``retry_backoff * n``. Default 0.25.
547
480
 
548
481
  Returns:
549
482
  ``True`` on 2xx response, ``False`` otherwise.
@@ -572,5 +505,11 @@ class LoopGain:
572
505
  include_per_iteration=include_per_iteration,
573
506
  )
574
507
  return send_payload(
575
- endpoint, token, payload, timeout=timeout, allow_insecure=allow_insecure
508
+ endpoint,
509
+ token,
510
+ payload,
511
+ timeout=timeout,
512
+ allow_insecure=allow_insecure,
513
+ retries=retries,
514
+ retry_backoff=retry_backoff,
576
515
  )
@@ -22,7 +22,9 @@ from __future__ import annotations
22
22
 
23
23
  import json
24
24
  import math
25
+ import socket
25
26
  import statistics
27
+ import time
26
28
  import urllib.error
27
29
  import urllib.request
28
30
  from datetime import datetime, timezone
@@ -35,12 +37,11 @@ def _safe_float(x: Any) -> Any:
35
37
 
36
38
  Standard JSON (RFC 8259) forbids Infinity and NaN literals. Python's
37
39
  json.dumps emits them by default, and strict parsers — including the
38
- Cloudflare-side receiver — reject the payload. gain_margin in particular
39
- is 1/max(Aβ_smooth) and goes to +inf whenever the smoothed gain is zero
40
- (e.g. a constant-error trajectory). values themselves can go to inf
41
- if a previous error is exactly zero. Collapsing to None keeps the
42
- dashboard's "no data" semantics intact instead of dropping the whole
43
- payload.
40
+ Cloudflare-side receiver — reject the payload. values can go to inf
41
+ if a previous error is exactly zero, so the convergence-profile summary
42
+ and per-iteration arrays are wrapped in this coercion. Collapsing to None
43
+ keeps the dashboard's "no data" semantics intact instead of dropping the
44
+ whole payload.
44
45
  """
45
46
  if isinstance(x, float) and not math.isfinite(x):
46
47
  return None
@@ -84,12 +85,14 @@ if TYPE_CHECKING:
84
85
 
85
86
 
86
87
  # Schema version is incremented when the payload format breaks compatibility.
87
- # v2 (2026-05-13) adds first_eta_prediction + first_eta_at_iteration for the
88
- # ETA Accuracy dashboard panel. v3 (2026-05-14) adds the optional
89
- # per_iteration block (capped trajectories) and the framework/loop_type/team
90
- # classification fields. Receiver remains backward-compatible: v1/v2 payloads
91
- # are still accepted (new fields default to None / NULL).
92
- SCHEMA_VERSION = 3
88
+ # v2 (2026-05-13) added an ETA-calibration block and v1 carried a gain_margin
89
+ # field; both were removed in v4 (2026-06-09) when ETA and gain margin were
90
+ # discontinued (neither fired reliably on real trajectories). v3 (2026-05-14)
91
+ # added the optional per_iteration block (capped trajectories) and the
92
+ # framework/loop_type/team classification fields. The receiver remains
93
+ # backward-compatible: older payloads are still accepted and the dropped
94
+ # fields are simply ignored.
95
+ SCHEMA_VERSION = 4
93
96
 
94
97
 
95
98
  # Library version sourced from loopgain._version so there's exactly one
@@ -174,7 +177,6 @@ def build_payload(
174
177
  "loop": {
175
178
  "outcome": result.outcome,
176
179
  "iterations_used": result.iterations_used,
177
- "gain_margin": _safe_float(result.gain_margin),
178
180
  "savings_vs_fixed_cap": result.savings_vs_fixed_cap,
179
181
  "convergence_profile_summary": profile_summary,
180
182
  "rollback_triggered": result.outcome in ("oscillating", "diverged"),
@@ -183,13 +185,6 @@ def build_payload(
183
185
  # (iterations_used-1-best_index) — the "Iteration Waste" view.
184
186
  # Privacy-safe: an integer position, no output/error content.
185
187
  "best_index": result.best_index,
186
- # v2: first computable eta snapshot, for ETA calibration dashboard.
187
- # Predicted total iterations = first_eta_at_iteration +
188
- # first_eta_prediction; compare to iterations_used to compute the
189
- # calibration error. Both are None when no prediction was made
190
- # (target_error=0, loop never looked convergent, etc.).
191
- "first_eta_prediction": result.first_eta_prediction,
192
- "first_eta_at_iteration": result.first_eta_at_iteration,
193
188
  },
194
189
  "thresholds": {
195
190
  "fast_converge": lg.thresholds.fast_converge,
@@ -218,18 +213,43 @@ def build_payload(
218
213
  return payload
219
214
 
220
215
 
216
+ def _is_transient(exc: BaseException) -> bool:
217
+ """Is this send failure worth retrying?
218
+
219
+ Transient = timeout, connection/DNS error, or a 5xx/429 from the server —
220
+ a later attempt might succeed. Deterministic failures (4xx other than 429,
221
+ a refused redirect) will never succeed on retry, so they are *not*
222
+ transient and we give up immediately.
223
+ """
224
+ if isinstance(exc, urllib.error.HTTPError): # subclass of URLError — check first
225
+ return exc.code >= 500 or exc.code == 429
226
+ return isinstance(exc, (TimeoutError, socket.timeout, urllib.error.URLError, OSError))
227
+
228
+
221
229
  def send_payload(
222
230
  endpoint: str,
223
231
  token: str,
224
232
  payload: dict[str, Any],
225
233
  timeout: float = 2.0,
226
234
  allow_insecure: bool = False,
235
+ retries: int = 2,
236
+ retry_backoff: float = 0.25,
227
237
  ) -> bool:
228
238
  """POST a telemetry payload to the given endpoint.
229
239
 
230
240
  Best-effort: errors are swallowed; never raises. Returns ``True`` if
231
241
  the server returned a 2xx status, ``False`` otherwise.
232
242
 
243
+ A single send is one HTTP POST with a ``timeout``-second deadline. The
244
+ warm round-trip to the hosted receiver is ~150 ms, so the default 2 s
245
+ timeout has wide headroom; the failure mode in practice is a *transient*
246
+ outlier (a cold database first-write, a momentary network blip) that
247
+ blows past it. Because a low-frequency caller may send only one aggregate
248
+ per run, a single dropped send loses that whole run's data — so a transient
249
+ failure is retried up to ``retries`` times with a short linear backoff.
250
+ Deterministic failures (bad token, malformed payload, refused redirect)
251
+ are *not* retried. Still best-effort throughout: the loop never raises.
252
+
233
253
  Args:
234
254
  endpoint: Telemetry receiver URL (e.g.,
235
255
  ``https://telemetry.loopgain.ai/v1/aggregate``). Must use
@@ -240,13 +260,18 @@ def send_payload(
240
260
  token: Bearer token issued by the receiver. Identifies the customer
241
261
  account; rotatable; not linked to any production secrets.
242
262
  payload: Dict from ``build_payload``.
243
- timeout: Per-request timeout in seconds. Default 2.0.
263
+ timeout: Per-attempt timeout in seconds. Default 2.0.
244
264
  allow_insecure: If ``True``, permit ``http://`` endpoints. Intended
245
265
  for local development against a self-hosted receiver on
246
266
  ``http://localhost``. Default ``False``.
267
+ retries: Number of *additional* attempts after the first if the send
268
+ fails transiently. Default 2 (so up to 3 attempts total). Set to
269
+ 0 to restore single-shot behavior.
270
+ retry_backoff: Base seconds to sleep between attempts; the nth retry
271
+ waits ``retry_backoff * n`` (0.25 s, 0.50 s, …). Default 0.25.
247
272
 
248
273
  Returns:
249
- ``True`` on 2xx response, ``False`` otherwise.
274
+ ``True`` on a 2xx response, ``False`` otherwise.
250
275
  """
251
276
  # Refuse to attach the bearer token to anything but http(s); silently
252
277
  # best-effort so a misconfigured endpoint can't break the user's loop.
@@ -263,23 +288,33 @@ def send_payload(
263
288
 
264
289
  try:
265
290
  body = json.dumps(payload).encode("utf-8")
266
- req = urllib.request.Request(
267
- endpoint,
268
- data=body,
269
- method="POST",
270
- headers={
271
- "Content-Type": "application/json",
272
- "Authorization": f"Bearer {token}",
273
- "User-Agent": f"loopgain/{LIBRARY_VERSION}",
274
- },
275
- )
276
- # Use the no-redirect seam so a malicious or misconfigured
277
- # endpoint can't 302 the bearer token to a different host.
278
- with _open_request(req, timeout) as resp:
279
- return 200 <= resp.status < 300
280
291
  except Exception:
281
- # Best-effort: never break the user's loop because telemetry failed.
282
- # Catches URLError, HTTPError, TimeoutError, OSError, plus the
283
- # ValueError that urllib raises for malformed URLs (e.g., missing scheme),
284
- # plus any JSON-encoding edge case in the payload.
292
+ # A payload that won't JSON-encode will never send don't retry.
285
293
  return False
294
+
295
+ req = urllib.request.Request(
296
+ endpoint,
297
+ data=body,
298
+ method="POST",
299
+ headers={
300
+ "Content-Type": "application/json",
301
+ "Authorization": f"Bearer {token}",
302
+ "User-Agent": f"loopgain/{LIBRARY_VERSION}",
303
+ },
304
+ )
305
+
306
+ attempts = max(1, retries + 1)
307
+ for i in range(attempts):
308
+ try:
309
+ # Use the no-redirect seam so a malicious or misconfigured
310
+ # endpoint can't 302 the bearer token to a different host.
311
+ with _open_request(req, timeout) as resp:
312
+ return 200 <= resp.status < 300
313
+ except Exception as exc:
314
+ # Best-effort: never break the user's loop because telemetry failed.
315
+ # Retry only transient failures, and only if attempts remain.
316
+ last = i == attempts - 1
317
+ if last or not _is_transient(exc):
318
+ return False
319
+ time.sleep(retry_backoff * (i + 1))
320
+ return False
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: loopgain
3
- Version: 0.4.2
3
+ Version: 0.5.0
4
4
  Summary: An open-source cost controller for AI agent loops. Stops a loop when it has actually converged and rolls back before it degrades — replacing the max_iterations guess with a real-time loop-gain (Aβ) monitor with five named threshold bands and best-so-far rollback.
5
5
  Author-email: Dave Fitzsimmons <hello@loopgain.ai>
6
6
  License: Apache-2.0
@@ -102,7 +102,6 @@ result = lg.result
102
102
  print(result.outcome) # "converged" | "oscillating" | "diverged" | "stalled" | "max_iterations"
103
103
  print(result.best_output) # the lowest-error iteration's output
104
104
  print(result.iterations_used)
105
- print(result.gain_margin) # 1 / max(Aβ_smooth)
106
105
  print(result.savings_vs_fixed_cap)
107
106
  ```
108
107
 
@@ -183,6 +182,7 @@ LoopGain saves money by stopping a loop once it stops improving — fewer iterat
183
182
 
184
183
  - **Savings depend on your workload.** Loops that usually succeed fast save the most (~96%); adversarial, failure-prone loops save less (~78–84%). The headline is a blend — run the benchmark on your own loops before quoting a number.
185
184
  - **LoopGain detects convergence, not correctness.** It stops when your error signal stops improving — which means more iterations won't help, *not* that the loop succeeded. On the benchmark this preserved quality (it rarely stopped early on a worse output; false-stop rate ≤4.5%), but a loop can stall with the error still above zero — a plateau at, say, 2 failing tests. So check `result.best_error` (or your own pass/fail) before you trust the output: if it plateaued short of your target, that's a quality gap LoopGain can't see, and a false stop that forces a rerun is the one way it eats into the savings. LoopGain decides *when to stop*; you decide *whether the answer is good enough*.
185
+ - **LoopGain is only as right as your verifier.** It acts on the error signal you give it. If your verifier reports zero errors, LoopGain trusts that and stops — so a verifier with blind spots can report success on an answer that is still wrong, and LoopGain will confidently stop there. This is not the plateau case above: the error reads zero and the loop looks like a clean success, so neither LoopGain nor its convergence signal can flag it. The quality of the stop is bounded by the quality of the check behind your error signal. Pair LoopGain with the strongest verifier you can afford at the stop — executable tests over a sampled subset, a schema or type check over a vibe, a held-out check the loop didn't optimize against.
186
186
 
187
187
  ---
188
188
 
@@ -212,17 +212,9 @@ Returns `False` once a terminal state fires.
212
212
 
213
213
  Current state name. One of `INIT`, `FAST_CONVERGE`, `CONVERGING`, `STALLING`, `OSCILLATING`, `DIVERGING`, `TARGET_MET`, `MAX_ITERATIONS`. The corresponding terminal `result.outcome` values are `converged`, `oscillating`, `diverged`, `stalled` (v0.2 trajectory mode only — STALLING terminating after 2 consecutive readings), `max_iterations`, or `in_progress`.
214
214
 
215
- ### `lg.eta -> int | None`
216
-
217
- Best-effort closed-form estimate of iterations remaining, exposed for instrumentation. Returns `None` whenever it isn't well-defined — which is most of the time on real, jump-dominated loops, so don't depend on it for control.
218
-
219
- ### `lg.gain_margin -> float | None`
220
-
221
- `1 / max(Aβ_smooth)`. `> 1` means stable headroom across the entire run.
222
-
223
215
  ### `lg.result -> LoopGainResult`
224
216
 
225
- Terminal result with `outcome`, `iterations_used`, `best_index`, `best_output`, `best_error`, `convergence_profile`, `error_history`, `gain_margin`, `savings_vs_fixed_cap`. Safe to call mid-loop.
217
+ Terminal result with `outcome`, `iterations_used`, `best_index`, `best_output`, `best_error`, `convergence_profile`, `error_history`, `savings_vs_fixed_cap`. Safe to call mid-loop.
226
218
 
227
219
  ### `lg.send_telemetry(endpoint, token, workload_id=None, timeout=2.0, allow_insecure=False, framework=None, loop_type=None, team=None, include_per_iteration=True) -> bool`
228
220
 
@@ -2,7 +2,7 @@
2
2
 
3
3
  Covers the five band transitions, the three canonical scenarios from the
4
4
  spec (converging / oscillating / diverging), TARGET_MET short-circuit,
5
- best-so-far buffer correctness, and ETA calibration.
5
+ and best-so-far buffer correctness.
6
6
  """
7
7
 
8
8
  from __future__ import annotations
@@ -179,109 +179,6 @@ def test_best_so_far_works_without_outputs():
179
179
  assert result.best_error == 2.0
180
180
 
181
181
 
182
- # ----- ETA prediction calibration -----
183
-
184
-
185
- def test_eta_prediction_calibration():
186
- """Synthetic monotonic decay: predicted iterations matches actual ±1."""
187
- target = 0.1
188
- ab = 0.5
189
- errors = _decay(ab, e0=100.0)
190
- # Iteration index where error first drops below target.
191
- actual_n = next(i for i, e in enumerate(errors) if e < target)
192
-
193
- lg = LoopGain(target_error=target, max_iterations=100)
194
- # Feed a few errors so smoothed Aβ stabilizes.
195
- lg.observe(errors[0])
196
- lg.observe(errors[1])
197
- lg.observe(errors[2])
198
- # ETA at this point should predict ~(actual_n - 2) more iterations.
199
- eta = lg.eta
200
- assert eta is not None
201
- assert abs(eta - (actual_n - 2)) <= 1, f"eta={eta}, expected ~{actual_n - 2}"
202
-
203
-
204
- def test_eta_none_when_not_converging():
205
- """ETA is None when Aβ_smooth >= 1 (non-converging)."""
206
- lg = LoopGain(target_error=0.5)
207
- for _ in range(3):
208
- lg.observe(10.0) # constant errors -> Aβ = 1
209
- if not lg.should_continue():
210
- break
211
- assert lg.eta is None
212
-
213
-
214
- def test_eta_none_when_target_is_zero():
215
- lg = LoopGain(target_error=0.0)
216
- lg.observe(100.0)
217
- lg.observe(50.0)
218
- assert lg.eta is None
219
-
220
-
221
- # ----- First-eta snapshot (for ETA Accuracy dashboard panel) -----
222
-
223
-
224
- def test_first_eta_snapshot_captured_during_converging_run():
225
- """Result carries the first non-None eta and the iter it was made at."""
226
- target = 0.1
227
- errors = _decay(0.5, e0=100.0)
228
- lg = LoopGain(target_error=target, max_iterations=100)
229
- for e in errors:
230
- lg.observe(e)
231
- if not lg.should_continue():
232
- break
233
-
234
- result = lg.result
235
- # First eta becomes computable on the 2nd observation (smoothed_history
236
- # exists, target > 0, current > target, Aβ_smooth < 1).
237
- assert result.first_eta_at_iteration == 2
238
- assert result.first_eta_prediction is not None
239
- assert result.first_eta_prediction > 0
240
- # Predicted total iterations should be within ±2 of actual (the prediction
241
- # is made early, before smoothing has fully settled).
242
- predicted_total = result.first_eta_at_iteration + result.first_eta_prediction
243
- assert abs(predicted_total - result.iterations_used) <= 2
244
-
245
-
246
- def test_first_eta_snapshot_none_when_target_is_zero():
247
- """No prediction is captured when target_error=0 (eta is always None)."""
248
- lg = LoopGain(target_error=0.0, max_iterations=5)
249
- for _ in range(5):
250
- lg.observe(10.0)
251
- result = lg.result
252
- assert result.first_eta_prediction is None
253
- assert result.first_eta_at_iteration is None
254
-
255
-
256
- def test_first_eta_snapshot_none_when_loop_never_converges():
257
- """Oscillating loop (Aβ ≈ 1) never produces a positive eta."""
258
- lg = LoopGain(target_error=0.5)
259
- for _ in range(5):
260
- lg.observe(10.0)
261
- if not lg.should_continue():
262
- break
263
- result = lg.result
264
- assert result.first_eta_prediction is None
265
- assert result.first_eta_at_iteration is None
266
-
267
-
268
- def test_first_eta_snapshot_is_idempotent():
269
- """Subsequent observations don't overwrite the first prediction."""
270
- target = 0.1
271
- errors = _decay(0.5, e0=100.0)
272
- lg = LoopGain(target_error=target, max_iterations=100)
273
- lg.observe(errors[0])
274
- lg.observe(errors[1])
275
- first = lg._first_eta_prediction
276
- first_iter = lg._first_eta_at_iteration
277
- assert first is not None
278
- # Run a few more iterations; the snapshot should not change.
279
- for e in errors[2:6]:
280
- lg.observe(e)
281
- assert lg._first_eta_prediction == first
282
- assert lg._first_eta_at_iteration == first_iter
283
-
284
-
285
182
  # ----- observe() input coercion -----
286
183
 
287
184
 
@@ -353,23 +250,6 @@ def test_max_iterations_with_converging_loop():
353
250
  assert not lg.should_continue()
354
251
 
355
252
 
356
- # ----- gain_margin -----
357
-
358
-
359
- def test_gain_margin_greater_than_one_for_converging():
360
- lg = LoopGain(max_iterations=10)
361
- for e in _decay(0.5)[:5]:
362
- lg.observe(e)
363
- gm = lg.gain_margin
364
- assert gm is not None
365
- assert gm > 1.0
366
-
367
-
368
- def test_gain_margin_none_before_observations():
369
- lg = LoopGain()
370
- assert lg.gain_margin is None
371
-
372
-
373
253
  # ----- result before any observations -----
374
254
 
375
255
 
@@ -130,8 +130,7 @@ def test_very_large_error_magnitudes():
130
130
  lg.observe(e)
131
131
  # Aβ = 0.4 → FAST_CONVERGE.
132
132
  assert lg.state in (FAST_CONVERGE, CONVERGING)
133
- assert lg.gain_margin is not None
134
- assert lg.gain_margin > 1.0
133
+ assert lg.result.convergence_profile # computed without overflow
135
134
 
136
135
 
137
136
  def test_zero_error_in_middle_of_run_anomalous():
@@ -245,19 +244,6 @@ def test_result_callable_mid_loop():
245
244
  assert r2.iterations_used == 3
246
245
 
247
246
 
248
- def test_eta_before_any_observation():
249
- """eta on a fresh instance is None, not a crash."""
250
- lg = LoopGain(target_error=0.5)
251
- assert lg.eta is None
252
-
253
-
254
- def test_gain_margin_with_single_observation():
255
- """Single observation has no Aβ yet → gain_margin is None."""
256
- lg = LoopGain()
257
- lg.observe(10.0)
258
- assert lg.gain_margin is None
259
-
260
-
261
247
  # ----- Mixed input types -----
262
248
 
263
249
 
@@ -53,7 +53,6 @@ def test_payload_loop_section_has_outcome_and_stats():
53
53
  loop = p["loop"]
54
54
  assert loop["outcome"] == "converged"
55
55
  assert loop["iterations_used"] == 4
56
- assert loop["gain_margin"] is not None
57
56
  assert loop["savings_vs_fixed_cap"] is not None
58
57
  assert "convergence_profile_summary" in loop
59
58
  assert "rollback_triggered" in loop
@@ -120,7 +119,7 @@ def test_payload_workload_id_optional():
120
119
 
121
120
 
122
121
  def test_payload_serializes_strict_json_for_constant_error_trajectory():
123
- """A constant-error trajectory pushes gain_margin to +inf (1/max()=1/0).
122
+ """A zero-error trajectory pushes to +inf (E(n)/E(n-1) with E(n-1)=0).
124
123
 
125
124
  Standard JSON forbids Infinity / NaN, and the receiver rejects payloads
126
125
  that include them. The build_payload sanitizer must coerce non-finite
@@ -135,8 +134,8 @@ def test_payload_serializes_strict_json_for_constant_error_trajectory():
135
134
  # Strict round-trip: allow_nan=False raises on inf/nan.
136
135
  encoded = json.dumps(p, allow_nan=False)
137
136
  decoded = json.loads(encoded)
138
- assert decoded["loop"]["gain_margin"] is None
139
- # Per-iteration values can also be non-finite (E(n)/E(n-1) with E(n-1)=0).
137
+ # Per-iteration values can be non-finite (E(n)/E(n-1) with E(n-1)=0)
138
+ # and the convergence-profile summary must stay finite-or-None too.
140
139
  for v in decoded["per_iteration"]["convergence_profile"]:
141
140
  assert v is None or isinstance(v, (int, float))
142
141
 
@@ -190,50 +189,24 @@ def test_payload_for_not_started_loop():
190
189
  assert p["loop"]["convergence_profile_summary"]["samples"] == 0
191
190
 
192
191
 
193
- # ----- v2 schema: ETA calibration fields -----
192
+ # ----- schema version -----
194
193
 
195
194
 
196
- def test_payload_schema_version_is_v3():
197
- """Schema bumped to v3 with the addition of per_iteration + classification."""
198
- assert SCHEMA_VERSION == 3
195
+ def test_payload_schema_version_is_v4():
196
+ """Schema bumped to v4 when ETA + gain_margin were removed from the payload."""
197
+ assert SCHEMA_VERSION == 4
199
198
  lg = _make_terminated_loop()
200
199
  p = build_payload(lg)
201
- assert p["schema_version"] == 3
200
+ assert p["schema_version"] == 4
202
201
 
203
202
 
204
- def test_payload_includes_first_eta_fields_when_loop_converged():
205
- """A converging loop produces a captured eta snapshot."""
203
+ def test_payload_loop_section_drops_eta_and_gain_margin():
204
+ """v4 no longer carries the discontinued ETA / gain_margin fields."""
206
205
  lg = _make_terminated_loop()
207
- p = build_payload(lg)
208
- loop = p["loop"]
209
- assert "first_eta_prediction" in loop
210
- assert "first_eta_at_iteration" in loop
211
- assert loop["first_eta_prediction"] is not None
212
- assert loop["first_eta_at_iteration"] is not None
213
- assert loop["first_eta_prediction"] > 0
214
- assert loop["first_eta_at_iteration"] >= 2
215
-
216
-
217
- def test_payload_first_eta_none_for_target_zero():
218
- """target_error=0 means eta is never computable; both fields are None."""
219
- lg = LoopGain(target_error=0.0, max_iterations=4)
220
- for _ in range(4):
221
- lg.observe(10.0)
222
- p = build_payload(lg)
223
- assert p["loop"]["first_eta_prediction"] is None
224
- assert p["loop"]["first_eta_at_iteration"] is None
225
-
226
-
227
- def test_payload_first_eta_none_for_diverging_loop():
228
- """A divergent loop never produces a positive eta."""
229
- lg = LoopGain(target_error=0.5, max_iterations=20)
230
- for e in [10.0, 12.0, 15.0, 20.0, 30.0]:
231
- if not lg.should_continue():
232
- break
233
- lg.observe(e)
234
- p = build_payload(lg)
235
- assert p["loop"]["first_eta_prediction"] is None
236
- assert p["loop"]["first_eta_at_iteration"] is None
206
+ loop = build_payload(lg)["loop"]
207
+ assert "gain_margin" not in loop
208
+ assert "first_eta_prediction" not in loop
209
+ assert "first_eta_at_iteration" not in loop
237
210
 
238
211
 
239
212
  # ----- send_payload behavior -----
@@ -660,3 +633,117 @@ def test_send_payload_refuses_redirects():
660
633
  req = urllib.request.Request("https://example.com/")
661
634
  with pytest.raises(urllib.error.HTTPError):
662
635
  method(req, io.BytesIO(b""), 302, "Found", {})
636
+
637
+
638
+ # ----- send_payload retry behavior (transient failures) -----
639
+
640
+ import socket as _socket
641
+ import urllib.error as _uerr
642
+
643
+ from loopgain import telemetry as _tele
644
+
645
+
646
+ class _OkResp:
647
+ status = 202
648
+
649
+ def __enter__(self):
650
+ return self
651
+
652
+ def __exit__(self, *args):
653
+ pass
654
+
655
+
656
+ def _retry_payload():
657
+ return build_payload(_make_terminated_loop(), workload_id="retry-test")
658
+
659
+
660
+ def test_send_payload_retries_transient_then_succeeds(monkeypatch):
661
+ """A transient failure (timeout) is retried; a later success returns True."""
662
+ calls = {"n": 0}
663
+
664
+ def flaky(req, timeout=None):
665
+ calls["n"] += 1
666
+ if calls["n"] < 3:
667
+ raise _socket.timeout("slow first attempts")
668
+ return _OkResp()
669
+
670
+ sleeps: list[float] = []
671
+ monkeypatch.setattr("loopgain.telemetry._open_request", flaky)
672
+ monkeypatch.setattr("loopgain.telemetry.time.sleep", lambda s: sleeps.append(s))
673
+
674
+ ok = send_payload("https://t.example/v1/aggregate", token="t", payload=_retry_payload())
675
+ assert ok is True
676
+ assert calls["n"] == 3 # two transient failures, third succeeds
677
+ assert sleeps == [0.25, 0.5] # linear backoff between attempts
678
+
679
+
680
+ def test_send_payload_gives_up_after_retries_on_persistent_5xx(monkeypatch):
681
+ """A persistent transient (503) exhausts retries and returns False."""
682
+ calls = {"n": 0}
683
+
684
+ def always_503(req, timeout=None):
685
+ calls["n"] += 1
686
+ raise _uerr.HTTPError("https://t.example", 503, "unavailable", {}, None)
687
+
688
+ monkeypatch.setattr("loopgain.telemetry._open_request", always_503)
689
+ monkeypatch.setattr("loopgain.telemetry.time.sleep", lambda s: None)
690
+
691
+ ok = send_payload("https://t.example/v1/aggregate", token="t", payload=_retry_payload(), retries=2)
692
+ assert ok is False
693
+ assert calls["n"] == 3 # 1 initial + 2 retries
694
+
695
+
696
+ def test_send_payload_does_not_retry_deterministic_4xx(monkeypatch):
697
+ """A 401 will never succeed on retry — fail fast, no backoff."""
698
+ calls = {"n": 0}
699
+ slept = {"n": 0}
700
+
701
+ def unauthorized(req, timeout=None):
702
+ calls["n"] += 1
703
+ raise _uerr.HTTPError("https://t.example", 401, "unauthorized", {}, None)
704
+
705
+ monkeypatch.setattr("loopgain.telemetry._open_request", unauthorized)
706
+ monkeypatch.setattr("loopgain.telemetry.time.sleep", lambda s: slept.__setitem__("n", slept["n"] + 1))
707
+
708
+ ok = send_payload("https://t.example/v1/aggregate", token="bad", payload=_retry_payload())
709
+ assert ok is False
710
+ assert calls["n"] == 1 # no retry on a deterministic 4xx
711
+ assert slept["n"] == 0
712
+
713
+
714
+ def test_send_payload_retries_zero_is_single_shot(monkeypatch):
715
+ """retries=0 restores the original single-attempt behavior."""
716
+ calls = {"n": 0}
717
+
718
+ def timeout(req, timeout=None):
719
+ calls["n"] += 1
720
+ raise TimeoutError()
721
+
722
+ monkeypatch.setattr("loopgain.telemetry._open_request", timeout)
723
+ monkeypatch.setattr("loopgain.telemetry.time.sleep", lambda s: None)
724
+
725
+ ok = send_payload("https://t.example/v1/aggregate", token="t", payload=_retry_payload(), retries=0)
726
+ assert ok is False
727
+ assert calls["n"] == 1
728
+
729
+
730
+ def test_send_payload_never_raises_on_unexpected_error(monkeypatch):
731
+ """A non-transient, unexpected error is swallowed (best-effort), no retry."""
732
+ def boom(req, timeout=None):
733
+ raise RuntimeError("unexpected")
734
+
735
+ monkeypatch.setattr("loopgain.telemetry._open_request", boom)
736
+ monkeypatch.setattr("loopgain.telemetry.time.sleep", lambda s: None)
737
+
738
+ assert send_payload("https://t.example/v1/aggregate", token="t", payload=_retry_payload()) is False
739
+
740
+
741
+ def test_is_transient_classification():
742
+ assert _tele._is_transient(TimeoutError()) is True
743
+ assert _tele._is_transient(_socket.timeout()) is True
744
+ assert _tele._is_transient(_uerr.URLError("dns")) is True
745
+ assert _tele._is_transient(_uerr.HTTPError("u", 503, "x", {}, None)) is True
746
+ assert _tele._is_transient(_uerr.HTTPError("u", 429, "x", {}, None)) is True
747
+ assert _tele._is_transient(_uerr.HTTPError("u", 400, "x", {}, None)) is False
748
+ assert _tele._is_transient(_uerr.HTTPError("u", 401, "x", {}, None)) is False
749
+ assert _tele._is_transient(RuntimeError("x")) is False
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes