loopgain 0.4.2__tar.gz → 0.5.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {loopgain-0.4.2 → loopgain-0.5.0}/PKG-INFO +3 -11
- {loopgain-0.4.2 → loopgain-0.5.0}/README.md +2 -10
- {loopgain-0.4.2 → loopgain-0.5.0}/loopgain/_version.py +1 -1
- {loopgain-0.4.2 → loopgain-0.5.0}/loopgain/core.py +16 -77
- {loopgain-0.4.2 → loopgain-0.5.0}/loopgain/telemetry.py +75 -40
- {loopgain-0.4.2 → loopgain-0.5.0}/loopgain.egg-info/PKG-INFO +3 -11
- {loopgain-0.4.2 → loopgain-0.5.0}/tests/test_core.py +1 -121
- {loopgain-0.4.2 → loopgain-0.5.0}/tests/test_stress.py +1 -15
- {loopgain-0.4.2 → loopgain-0.5.0}/tests/test_telemetry.py +128 -41
- {loopgain-0.4.2 → loopgain-0.5.0}/LICENSE +0 -0
- {loopgain-0.4.2 → loopgain-0.5.0}/loopgain/__init__.py +0 -0
- {loopgain-0.4.2 → loopgain-0.5.0}/loopgain/__main__.py +0 -0
- {loopgain-0.4.2 → loopgain-0.5.0}/loopgain/classifier.py +0 -0
- {loopgain-0.4.2 → loopgain-0.5.0}/loopgain/cli.py +0 -0
- {loopgain-0.4.2 → loopgain-0.5.0}/loopgain/funnel.py +0 -0
- {loopgain-0.4.2 → loopgain-0.5.0}/loopgain/integrations/__init__.py +0 -0
- {loopgain-0.4.2 → loopgain-0.5.0}/loopgain/integrations/autogen.py +0 -0
- {loopgain-0.4.2 → loopgain-0.5.0}/loopgain/integrations/claude_agent_sdk.py +0 -0
- {loopgain-0.4.2 → loopgain-0.5.0}/loopgain/integrations/crewai.py +0 -0
- {loopgain-0.4.2 → loopgain-0.5.0}/loopgain/integrations/langchain.py +0 -0
- {loopgain-0.4.2 → loopgain-0.5.0}/loopgain/integrations/langgraph.py +0 -0
- {loopgain-0.4.2 → loopgain-0.5.0}/loopgain/integrations/openai_agents.py +0 -0
- {loopgain-0.4.2 → loopgain-0.5.0}/loopgain.egg-info/SOURCES.txt +0 -0
- {loopgain-0.4.2 → loopgain-0.5.0}/loopgain.egg-info/dependency_links.txt +0 -0
- {loopgain-0.4.2 → loopgain-0.5.0}/loopgain.egg-info/entry_points.txt +0 -0
- {loopgain-0.4.2 → loopgain-0.5.0}/loopgain.egg-info/requires.txt +0 -0
- {loopgain-0.4.2 → loopgain-0.5.0}/loopgain.egg-info/top_level.txt +0 -0
- {loopgain-0.4.2 → loopgain-0.5.0}/pyproject.toml +0 -0
- {loopgain-0.4.2 → loopgain-0.5.0}/setup.cfg +0 -0
- {loopgain-0.4.2 → loopgain-0.5.0}/tests/test_classifier_mock_validation.py +0 -0
- {loopgain-0.4.2 → loopgain-0.5.0}/tests/test_classifier_synthetic.py +0 -0
- {loopgain-0.4.2 → loopgain-0.5.0}/tests/test_funnel.py +0 -0
- {loopgain-0.4.2 → loopgain-0.5.0}/tests/test_integrations.py +0 -0
- {loopgain-0.4.2 → loopgain-0.5.0}/tests/test_termination_safety.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: loopgain
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.5.0
|
|
4
4
|
Summary: An open-source cost controller for AI agent loops. Stops a loop when it has actually converged and rolls back before it degrades — replacing the max_iterations guess with a real-time loop-gain (Aβ) monitor with five named threshold bands and best-so-far rollback.
|
|
5
5
|
Author-email: Dave Fitzsimmons <hello@loopgain.ai>
|
|
6
6
|
License: Apache-2.0
|
|
@@ -102,7 +102,6 @@ result = lg.result
|
|
|
102
102
|
print(result.outcome) # "converged" | "oscillating" | "diverged" | "stalled" | "max_iterations"
|
|
103
103
|
print(result.best_output) # the lowest-error iteration's output
|
|
104
104
|
print(result.iterations_used)
|
|
105
|
-
print(result.gain_margin) # 1 / max(Aβ_smooth)
|
|
106
105
|
print(result.savings_vs_fixed_cap)
|
|
107
106
|
```
|
|
108
107
|
|
|
@@ -183,6 +182,7 @@ LoopGain saves money by stopping a loop once it stops improving — fewer iterat
|
|
|
183
182
|
|
|
184
183
|
- **Savings depend on your workload.** Loops that usually succeed fast save the most (~96%); adversarial, failure-prone loops save less (~78–84%). The headline is a blend — run the benchmark on your own loops before quoting a number.
|
|
185
184
|
- **LoopGain detects convergence, not correctness.** It stops when your error signal stops improving — which means more iterations won't help, *not* that the loop succeeded. On the benchmark this preserved quality (it rarely stopped early on a worse output; false-stop rate ≤4.5%), but a loop can stall with the error still above zero — a plateau at, say, 2 failing tests. So check `result.best_error` (or your own pass/fail) before you trust the output: if it plateaued short of your target, that's a quality gap LoopGain can't see, and a false stop that forces a rerun is the one way it eats into the savings. LoopGain decides *when to stop*; you decide *whether the answer is good enough*.
|
|
185
|
+
- **LoopGain is only as right as your verifier.** It acts on the error signal you give it. If your verifier reports zero errors, LoopGain trusts that and stops — so a verifier with blind spots can report success on an answer that is still wrong, and LoopGain will confidently stop there. This is not the plateau case above: the error reads zero and the loop looks like a clean success, so neither LoopGain nor its convergence signal can flag it. The quality of the stop is bounded by the quality of the check behind your error signal. Pair LoopGain with the strongest verifier you can afford at the stop — executable tests over a sampled subset, a schema or type check over a vibe, a held-out check the loop didn't optimize against.
|
|
186
186
|
|
|
187
187
|
---
|
|
188
188
|
|
|
@@ -212,17 +212,9 @@ Returns `False` once a terminal state fires.
|
|
|
212
212
|
|
|
213
213
|
Current state name. One of `INIT`, `FAST_CONVERGE`, `CONVERGING`, `STALLING`, `OSCILLATING`, `DIVERGING`, `TARGET_MET`, `MAX_ITERATIONS`. The corresponding terminal `result.outcome` values are `converged`, `oscillating`, `diverged`, `stalled` (v0.2 trajectory mode only — STALLING terminating after 2 consecutive readings), `max_iterations`, or `in_progress`.
|
|
214
214
|
|
|
215
|
-
### `lg.eta -> int | None`
|
|
216
|
-
|
|
217
|
-
Best-effort closed-form estimate of iterations remaining, exposed for instrumentation. Returns `None` whenever it isn't well-defined — which is most of the time on real, jump-dominated loops, so don't depend on it for control.
|
|
218
|
-
|
|
219
|
-
### `lg.gain_margin -> float | None`
|
|
220
|
-
|
|
221
|
-
`1 / max(Aβ_smooth)`. `> 1` means stable headroom across the entire run.
|
|
222
|
-
|
|
223
215
|
### `lg.result -> LoopGainResult`
|
|
224
216
|
|
|
225
|
-
Terminal result with `outcome`, `iterations_used`, `best_index`, `best_output`, `best_error`, `convergence_profile`, `error_history`, `
|
|
217
|
+
Terminal result with `outcome`, `iterations_used`, `best_index`, `best_output`, `best_error`, `convergence_profile`, `error_history`, `savings_vs_fixed_cap`. Safe to call mid-loop.
|
|
226
218
|
|
|
227
219
|
### `lg.send_telemetry(endpoint, token, workload_id=None, timeout=2.0, allow_insecure=False, framework=None, loop_type=None, team=None, include_per_iteration=True) -> bool`
|
|
228
220
|
|
|
@@ -53,7 +53,6 @@ result = lg.result
|
|
|
53
53
|
print(result.outcome) # "converged" | "oscillating" | "diverged" | "stalled" | "max_iterations"
|
|
54
54
|
print(result.best_output) # the lowest-error iteration's output
|
|
55
55
|
print(result.iterations_used)
|
|
56
|
-
print(result.gain_margin) # 1 / max(Aβ_smooth)
|
|
57
56
|
print(result.savings_vs_fixed_cap)
|
|
58
57
|
```
|
|
59
58
|
|
|
@@ -134,6 +133,7 @@ LoopGain saves money by stopping a loop once it stops improving — fewer iterat
|
|
|
134
133
|
|
|
135
134
|
- **Savings depend on your workload.** Loops that usually succeed fast save the most (~96%); adversarial, failure-prone loops save less (~78–84%). The headline is a blend — run the benchmark on your own loops before quoting a number.
|
|
136
135
|
- **LoopGain detects convergence, not correctness.** It stops when your error signal stops improving — which means more iterations won't help, *not* that the loop succeeded. On the benchmark this preserved quality (it rarely stopped early on a worse output; false-stop rate ≤4.5%), but a loop can stall with the error still above zero — a plateau at, say, 2 failing tests. So check `result.best_error` (or your own pass/fail) before you trust the output: if it plateaued short of your target, that's a quality gap LoopGain can't see, and a false stop that forces a rerun is the one way it eats into the savings. LoopGain decides *when to stop*; you decide *whether the answer is good enough*.
|
|
136
|
+
- **LoopGain is only as right as your verifier.** It acts on the error signal you give it. If your verifier reports zero errors, LoopGain trusts that and stops — so a verifier with blind spots can report success on an answer that is still wrong, and LoopGain will confidently stop there. This is not the plateau case above: the error reads zero and the loop looks like a clean success, so neither LoopGain nor its convergence signal can flag it. The quality of the stop is bounded by the quality of the check behind your error signal. Pair LoopGain with the strongest verifier you can afford at the stop — executable tests over a sampled subset, a schema or type check over a vibe, a held-out check the loop didn't optimize against.
|
|
137
137
|
|
|
138
138
|
---
|
|
139
139
|
|
|
@@ -163,17 +163,9 @@ Returns `False` once a terminal state fires.
|
|
|
163
163
|
|
|
164
164
|
Current state name. One of `INIT`, `FAST_CONVERGE`, `CONVERGING`, `STALLING`, `OSCILLATING`, `DIVERGING`, `TARGET_MET`, `MAX_ITERATIONS`. The corresponding terminal `result.outcome` values are `converged`, `oscillating`, `diverged`, `stalled` (v0.2 trajectory mode only — STALLING terminating after 2 consecutive readings), `max_iterations`, or `in_progress`.
|
|
165
165
|
|
|
166
|
-
### `lg.eta -> int | None`
|
|
167
|
-
|
|
168
|
-
Best-effort closed-form estimate of iterations remaining, exposed for instrumentation. Returns `None` whenever it isn't well-defined — which is most of the time on real, jump-dominated loops, so don't depend on it for control.
|
|
169
|
-
|
|
170
|
-
### `lg.gain_margin -> float | None`
|
|
171
|
-
|
|
172
|
-
`1 / max(Aβ_smooth)`. `> 1` means stable headroom across the entire run.
|
|
173
|
-
|
|
174
166
|
### `lg.result -> LoopGainResult`
|
|
175
167
|
|
|
176
|
-
Terminal result with `outcome`, `iterations_used`, `best_index`, `best_output`, `best_error`, `convergence_profile`, `error_history`, `
|
|
168
|
+
Terminal result with `outcome`, `iterations_used`, `best_index`, `best_output`, `best_error`, `convergence_profile`, `error_history`, `savings_vs_fixed_cap`. Safe to call mid-loop.
|
|
177
169
|
|
|
178
170
|
### `lg.send_telemetry(endpoint, token, workload_id=None, timeout=2.0, allow_insecure=False, framework=None, loop_type=None, team=None, include_per_iteration=True) -> bool`
|
|
179
171
|
|
|
@@ -9,8 +9,7 @@ real-time loop-gain monitor that classifies the loop into one of five
|
|
|
9
9
|
named states and decides whether to continue, stop, or roll back.
|
|
10
10
|
|
|
11
11
|
The math is foundational EE control theory. The product layer is the
|
|
12
|
-
threshold bands, the best-so-far buffer, the
|
|
13
|
-
clean Python API.
|
|
12
|
+
threshold bands, the best-so-far buffer, and the clean Python API.
|
|
14
13
|
|
|
15
14
|
License: Apache-2.0
|
|
16
15
|
"""
|
|
@@ -121,28 +120,10 @@ class LoopGainResult:
|
|
|
121
120
|
error_history: list[float] = field(default_factory=list)
|
|
122
121
|
"""All observed error magnitudes, in order."""
|
|
123
122
|
|
|
124
|
-
gain_margin: Optional[float] = None
|
|
125
|
-
"""``1 / max(Aβ_smooth)``. > 1 means stable headroom; < 1 means the
|
|
126
|
-
loop crossed into oscillation/divergence at some point."""
|
|
127
|
-
|
|
128
123
|
savings_vs_fixed_cap: Optional[int] = None
|
|
129
124
|
"""Iterations saved versus the assumed fixed cap (default 10).
|
|
130
125
|
Zero if the loop hit ``max_iterations``; otherwise non-negative."""
|
|
131
126
|
|
|
132
|
-
first_eta_prediction: Optional[int] = None
|
|
133
|
-
"""First non-None ``eta`` snapshot captured during the loop —
|
|
134
|
-
the predicted iterations-remaining at the moment the prediction
|
|
135
|
-
became computable. ``None`` if no prediction was ever made
|
|
136
|
-
(e.g., ``target_error`` is ``None`` or ``0`` so the prediction
|
|
137
|
-
formula is undefined, loop never converged toward target, or the
|
|
138
|
-
loop terminated before two observations)."""
|
|
139
|
-
|
|
140
|
-
first_eta_at_iteration: Optional[int] = None
|
|
141
|
-
"""Iteration count when ``first_eta_prediction`` was captured.
|
|
142
|
-
``None`` if no prediction was ever made. Predicted *total*
|
|
143
|
-
iterations = ``first_eta_at_iteration + first_eta_prediction``,
|
|
144
|
-
comparable to ``iterations_used`` for calibration."""
|
|
145
|
-
|
|
146
127
|
|
|
147
128
|
class LoopGain:
|
|
148
129
|
"""Barkhausen stability monitor for AI agent loops.
|
|
@@ -239,8 +220,6 @@ class LoopGain:
|
|
|
239
220
|
self._state: str = INIT
|
|
240
221
|
self._state_history: list[str] = []
|
|
241
222
|
self._terminal: bool = False
|
|
242
|
-
self._first_eta_prediction: Optional[int] = None
|
|
243
|
-
self._first_eta_at_iteration: Optional[int] = None
|
|
244
223
|
|
|
245
224
|
# Opt-in anonymous funnel telemetry (see loopgain.funnel). No-op
|
|
246
225
|
# unless the user has explicitly opted in; fully fail-silent and
|
|
@@ -348,17 +327,6 @@ class LoopGain:
|
|
|
348
327
|
self._state = MAX_ITERATIONS
|
|
349
328
|
self._terminal = True
|
|
350
329
|
|
|
351
|
-
# Snapshot the first computable eta prediction for calibration.
|
|
352
|
-
# eta is None until smoothing settles and the loop looks convergent;
|
|
353
|
-
# we capture the *first* value it produces and the iteration it was
|
|
354
|
-
# produced at, so predicted_total = at_iter + eta is comparable to
|
|
355
|
-
# iterations_used.
|
|
356
|
-
if self._first_eta_prediction is None:
|
|
357
|
-
eta_now = self.eta
|
|
358
|
-
if eta_now is not None and eta_now > 0:
|
|
359
|
-
self._first_eta_prediction = eta_now
|
|
360
|
-
self._first_eta_at_iteration = len(self._error_history)
|
|
361
|
-
|
|
362
330
|
# Funnel telemetry: if this observation drove the loop terminal
|
|
363
331
|
# (oscillating / diverging / stalled / max-iterations), count the
|
|
364
332
|
# coarse outcome. The TARGET_MET case is handled at its early return.
|
|
@@ -382,46 +350,6 @@ class LoopGain:
|
|
|
382
350
|
"""Current state name."""
|
|
383
351
|
return self._state
|
|
384
352
|
|
|
385
|
-
@property
|
|
386
|
-
def eta(self) -> Optional[int]:
|
|
387
|
-
"""Predicted iterations remaining to reach ``target_error``.
|
|
388
|
-
|
|
389
|
-
Closed-form Barkhausen prediction:
|
|
390
|
-
|
|
391
|
-
n_remaining = log(E_target / E_current) / log(Aβ_smooth)
|
|
392
|
-
|
|
393
|
-
Returns ``None`` when the prediction isn't well-defined:
|
|
394
|
-
no Aβ yet, ``target_error`` is ``None`` (no target to predict
|
|
395
|
-
against) or ``0`` (the formula has a log-of-zero singularity),
|
|
396
|
-
target already met, or ``Aβ_smooth >= 1`` (non-converging gain).
|
|
397
|
-
"""
|
|
398
|
-
if not self._smoothed_history or not self._error_history:
|
|
399
|
-
return None
|
|
400
|
-
if self.target_error is None or self.target_error <= 0:
|
|
401
|
-
return None
|
|
402
|
-
e_current = self._error_history[-1]
|
|
403
|
-
if e_current <= self.target_error:
|
|
404
|
-
return 0
|
|
405
|
-
ab_smooth = self._smoothed_history[-1]
|
|
406
|
-
if ab_smooth >= 1.0 or ab_smooth <= 0:
|
|
407
|
-
return None
|
|
408
|
-
n = math.log(self.target_error / e_current) / math.log(ab_smooth)
|
|
409
|
-
return max(0, math.ceil(n))
|
|
410
|
-
|
|
411
|
-
@property
|
|
412
|
-
def gain_margin(self) -> Optional[float]:
|
|
413
|
-
"""Gain margin ``GM = 1 / max(Aβ_smooth)``.
|
|
414
|
-
|
|
415
|
-
``GM > 1`` means the loop never crossed into oscillation. The
|
|
416
|
-
larger, the more headroom. Returns ``None`` if no Aβ data yet.
|
|
417
|
-
"""
|
|
418
|
-
if not self._smoothed_history:
|
|
419
|
-
return None
|
|
420
|
-
max_g = max(self._smoothed_history)
|
|
421
|
-
if max_g == 0:
|
|
422
|
-
return float("inf")
|
|
423
|
-
return 1.0 / max_g
|
|
424
|
-
|
|
425
353
|
@property
|
|
426
354
|
def result(self) -> LoopGainResult:
|
|
427
355
|
"""Construct the terminal result. Safe to call any time."""
|
|
@@ -468,10 +396,7 @@ class LoopGain:
|
|
|
468
396
|
best_error=best_error,
|
|
469
397
|
convergence_profile=list(self._smoothed_history),
|
|
470
398
|
error_history=list(self._error_history),
|
|
471
|
-
gain_margin=self.gain_margin,
|
|
472
399
|
savings_vs_fixed_cap=savings,
|
|
473
|
-
first_eta_prediction=self._first_eta_prediction,
|
|
474
|
-
first_eta_at_iteration=self._first_eta_at_iteration,
|
|
475
400
|
)
|
|
476
401
|
|
|
477
402
|
# ----- Internal helpers -----
|
|
@@ -514,6 +439,8 @@ class LoopGain:
|
|
|
514
439
|
loop_type: Optional[str] = None,
|
|
515
440
|
team: Optional[str] = None,
|
|
516
441
|
include_per_iteration: bool = True,
|
|
442
|
+
retries: int = 2,
|
|
443
|
+
retry_backoff: float = 0.25,
|
|
517
444
|
) -> bool:
|
|
518
445
|
"""Send anonymized telemetry to a receiver endpoint.
|
|
519
446
|
|
|
@@ -544,6 +471,12 @@ class LoopGain:
|
|
|
544
471
|
per-iteration Aβ + error trajectories (capped) so the
|
|
545
472
|
dashboard's Loop Detail scrubber works. Set ``False`` to
|
|
546
473
|
send only aggregate summary stats.
|
|
474
|
+
retries: Additional attempts if a send fails *transiently*
|
|
475
|
+
(timeout, connection error, 5xx/429). Default 2 (up to 3
|
|
476
|
+
attempts). Set to 0 for single-shot. Deterministic failures
|
|
477
|
+
(bad token, etc.) are never retried.
|
|
478
|
+
retry_backoff: Base seconds between attempts; the nth retry waits
|
|
479
|
+
``retry_backoff * n``. Default 0.25.
|
|
547
480
|
|
|
548
481
|
Returns:
|
|
549
482
|
``True`` on 2xx response, ``False`` otherwise.
|
|
@@ -572,5 +505,11 @@ class LoopGain:
|
|
|
572
505
|
include_per_iteration=include_per_iteration,
|
|
573
506
|
)
|
|
574
507
|
return send_payload(
|
|
575
|
-
endpoint,
|
|
508
|
+
endpoint,
|
|
509
|
+
token,
|
|
510
|
+
payload,
|
|
511
|
+
timeout=timeout,
|
|
512
|
+
allow_insecure=allow_insecure,
|
|
513
|
+
retries=retries,
|
|
514
|
+
retry_backoff=retry_backoff,
|
|
576
515
|
)
|
|
@@ -22,7 +22,9 @@ from __future__ import annotations
|
|
|
22
22
|
|
|
23
23
|
import json
|
|
24
24
|
import math
|
|
25
|
+
import socket
|
|
25
26
|
import statistics
|
|
27
|
+
import time
|
|
26
28
|
import urllib.error
|
|
27
29
|
import urllib.request
|
|
28
30
|
from datetime import datetime, timezone
|
|
@@ -35,12 +37,11 @@ def _safe_float(x: Any) -> Any:
|
|
|
35
37
|
|
|
36
38
|
Standard JSON (RFC 8259) forbids Infinity and NaN literals. Python's
|
|
37
39
|
json.dumps emits them by default, and strict parsers — including the
|
|
38
|
-
Cloudflare-side receiver — reject the payload.
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
payload.
|
|
40
|
+
Cloudflare-side receiver — reject the payload. Aβ values can go to inf
|
|
41
|
+
if a previous error is exactly zero, so the convergence-profile summary
|
|
42
|
+
and per-iteration arrays are wrapped in this coercion. Collapsing to None
|
|
43
|
+
keeps the dashboard's "no data" semantics intact instead of dropping the
|
|
44
|
+
whole payload.
|
|
44
45
|
"""
|
|
45
46
|
if isinstance(x, float) and not math.isfinite(x):
|
|
46
47
|
return None
|
|
@@ -84,12 +85,14 @@ if TYPE_CHECKING:
|
|
|
84
85
|
|
|
85
86
|
|
|
86
87
|
# Schema version is incremented when the payload format breaks compatibility.
|
|
87
|
-
# v2 (2026-05-13)
|
|
88
|
-
#
|
|
89
|
-
#
|
|
90
|
-
#
|
|
91
|
-
#
|
|
92
|
-
|
|
88
|
+
# v2 (2026-05-13) added an ETA-calibration block and v1 carried a gain_margin
|
|
89
|
+
# field; both were removed in v4 (2026-06-09) when ETA and gain margin were
|
|
90
|
+
# discontinued (neither fired reliably on real trajectories). v3 (2026-05-14)
|
|
91
|
+
# added the optional per_iteration block (capped trajectories) and the
|
|
92
|
+
# framework/loop_type/team classification fields. The receiver remains
|
|
93
|
+
# backward-compatible: older payloads are still accepted and the dropped
|
|
94
|
+
# fields are simply ignored.
|
|
95
|
+
SCHEMA_VERSION = 4
|
|
93
96
|
|
|
94
97
|
|
|
95
98
|
# Library version sourced from loopgain._version so there's exactly one
|
|
@@ -174,7 +177,6 @@ def build_payload(
|
|
|
174
177
|
"loop": {
|
|
175
178
|
"outcome": result.outcome,
|
|
176
179
|
"iterations_used": result.iterations_used,
|
|
177
|
-
"gain_margin": _safe_float(result.gain_margin),
|
|
178
180
|
"savings_vs_fixed_cap": result.savings_vs_fixed_cap,
|
|
179
181
|
"convergence_profile_summary": profile_summary,
|
|
180
182
|
"rollback_triggered": result.outcome in ("oscillating", "diverged"),
|
|
@@ -183,13 +185,6 @@ def build_payload(
|
|
|
183
185
|
# (iterations_used-1-best_index) — the "Iteration Waste" view.
|
|
184
186
|
# Privacy-safe: an integer position, no output/error content.
|
|
185
187
|
"best_index": result.best_index,
|
|
186
|
-
# v2: first computable eta snapshot, for ETA calibration dashboard.
|
|
187
|
-
# Predicted total iterations = first_eta_at_iteration +
|
|
188
|
-
# first_eta_prediction; compare to iterations_used to compute the
|
|
189
|
-
# calibration error. Both are None when no prediction was made
|
|
190
|
-
# (target_error=0, loop never looked convergent, etc.).
|
|
191
|
-
"first_eta_prediction": result.first_eta_prediction,
|
|
192
|
-
"first_eta_at_iteration": result.first_eta_at_iteration,
|
|
193
188
|
},
|
|
194
189
|
"thresholds": {
|
|
195
190
|
"fast_converge": lg.thresholds.fast_converge,
|
|
@@ -218,18 +213,43 @@ def build_payload(
|
|
|
218
213
|
return payload
|
|
219
214
|
|
|
220
215
|
|
|
216
|
+
def _is_transient(exc: BaseException) -> bool:
|
|
217
|
+
"""Is this send failure worth retrying?
|
|
218
|
+
|
|
219
|
+
Transient = timeout, connection/DNS error, or a 5xx/429 from the server —
|
|
220
|
+
a later attempt might succeed. Deterministic failures (4xx other than 429,
|
|
221
|
+
a refused redirect) will never succeed on retry, so they are *not*
|
|
222
|
+
transient and we give up immediately.
|
|
223
|
+
"""
|
|
224
|
+
if isinstance(exc, urllib.error.HTTPError): # subclass of URLError — check first
|
|
225
|
+
return exc.code >= 500 or exc.code == 429
|
|
226
|
+
return isinstance(exc, (TimeoutError, socket.timeout, urllib.error.URLError, OSError))
|
|
227
|
+
|
|
228
|
+
|
|
221
229
|
def send_payload(
|
|
222
230
|
endpoint: str,
|
|
223
231
|
token: str,
|
|
224
232
|
payload: dict[str, Any],
|
|
225
233
|
timeout: float = 2.0,
|
|
226
234
|
allow_insecure: bool = False,
|
|
235
|
+
retries: int = 2,
|
|
236
|
+
retry_backoff: float = 0.25,
|
|
227
237
|
) -> bool:
|
|
228
238
|
"""POST a telemetry payload to the given endpoint.
|
|
229
239
|
|
|
230
240
|
Best-effort: errors are swallowed; never raises. Returns ``True`` if
|
|
231
241
|
the server returned a 2xx status, ``False`` otherwise.
|
|
232
242
|
|
|
243
|
+
A single send is one HTTP POST with a ``timeout``-second deadline. The
|
|
244
|
+
warm round-trip to the hosted receiver is ~150 ms, so the default 2 s
|
|
245
|
+
timeout has wide headroom; the failure mode in practice is a *transient*
|
|
246
|
+
outlier (a cold database first-write, a momentary network blip) that
|
|
247
|
+
blows past it. Because a low-frequency caller may send only one aggregate
|
|
248
|
+
per run, a single dropped send loses that whole run's data — so a transient
|
|
249
|
+
failure is retried up to ``retries`` times with a short linear backoff.
|
|
250
|
+
Deterministic failures (bad token, malformed payload, refused redirect)
|
|
251
|
+
are *not* retried. Still best-effort throughout: the loop never raises.
|
|
252
|
+
|
|
233
253
|
Args:
|
|
234
254
|
endpoint: Telemetry receiver URL (e.g.,
|
|
235
255
|
``https://telemetry.loopgain.ai/v1/aggregate``). Must use
|
|
@@ -240,13 +260,18 @@ def send_payload(
|
|
|
240
260
|
token: Bearer token issued by the receiver. Identifies the customer
|
|
241
261
|
account; rotatable; not linked to any production secrets.
|
|
242
262
|
payload: Dict from ``build_payload``.
|
|
243
|
-
timeout: Per-
|
|
263
|
+
timeout: Per-attempt timeout in seconds. Default 2.0.
|
|
244
264
|
allow_insecure: If ``True``, permit ``http://`` endpoints. Intended
|
|
245
265
|
for local development against a self-hosted receiver on
|
|
246
266
|
``http://localhost``. Default ``False``.
|
|
267
|
+
retries: Number of *additional* attempts after the first if the send
|
|
268
|
+
fails transiently. Default 2 (so up to 3 attempts total). Set to
|
|
269
|
+
0 to restore single-shot behavior.
|
|
270
|
+
retry_backoff: Base seconds to sleep between attempts; the nth retry
|
|
271
|
+
waits ``retry_backoff * n`` (0.25 s, 0.50 s, …). Default 0.25.
|
|
247
272
|
|
|
248
273
|
Returns:
|
|
249
|
-
``True`` on 2xx response, ``False`` otherwise.
|
|
274
|
+
``True`` on a 2xx response, ``False`` otherwise.
|
|
250
275
|
"""
|
|
251
276
|
# Refuse to attach the bearer token to anything but http(s); silently
|
|
252
277
|
# best-effort so a misconfigured endpoint can't break the user's loop.
|
|
@@ -263,23 +288,33 @@ def send_payload(
|
|
|
263
288
|
|
|
264
289
|
try:
|
|
265
290
|
body = json.dumps(payload).encode("utf-8")
|
|
266
|
-
req = urllib.request.Request(
|
|
267
|
-
endpoint,
|
|
268
|
-
data=body,
|
|
269
|
-
method="POST",
|
|
270
|
-
headers={
|
|
271
|
-
"Content-Type": "application/json",
|
|
272
|
-
"Authorization": f"Bearer {token}",
|
|
273
|
-
"User-Agent": f"loopgain/{LIBRARY_VERSION}",
|
|
274
|
-
},
|
|
275
|
-
)
|
|
276
|
-
# Use the no-redirect seam so a malicious or misconfigured
|
|
277
|
-
# endpoint can't 302 the bearer token to a different host.
|
|
278
|
-
with _open_request(req, timeout) as resp:
|
|
279
|
-
return 200 <= resp.status < 300
|
|
280
291
|
except Exception:
|
|
281
|
-
#
|
|
282
|
-
# Catches URLError, HTTPError, TimeoutError, OSError, plus the
|
|
283
|
-
# ValueError that urllib raises for malformed URLs (e.g., missing scheme),
|
|
284
|
-
# plus any JSON-encoding edge case in the payload.
|
|
292
|
+
# A payload that won't JSON-encode will never send — don't retry.
|
|
285
293
|
return False
|
|
294
|
+
|
|
295
|
+
req = urllib.request.Request(
|
|
296
|
+
endpoint,
|
|
297
|
+
data=body,
|
|
298
|
+
method="POST",
|
|
299
|
+
headers={
|
|
300
|
+
"Content-Type": "application/json",
|
|
301
|
+
"Authorization": f"Bearer {token}",
|
|
302
|
+
"User-Agent": f"loopgain/{LIBRARY_VERSION}",
|
|
303
|
+
},
|
|
304
|
+
)
|
|
305
|
+
|
|
306
|
+
attempts = max(1, retries + 1)
|
|
307
|
+
for i in range(attempts):
|
|
308
|
+
try:
|
|
309
|
+
# Use the no-redirect seam so a malicious or misconfigured
|
|
310
|
+
# endpoint can't 302 the bearer token to a different host.
|
|
311
|
+
with _open_request(req, timeout) as resp:
|
|
312
|
+
return 200 <= resp.status < 300
|
|
313
|
+
except Exception as exc:
|
|
314
|
+
# Best-effort: never break the user's loop because telemetry failed.
|
|
315
|
+
# Retry only transient failures, and only if attempts remain.
|
|
316
|
+
last = i == attempts - 1
|
|
317
|
+
if last or not _is_transient(exc):
|
|
318
|
+
return False
|
|
319
|
+
time.sleep(retry_backoff * (i + 1))
|
|
320
|
+
return False
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: loopgain
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.5.0
|
|
4
4
|
Summary: An open-source cost controller for AI agent loops. Stops a loop when it has actually converged and rolls back before it degrades — replacing the max_iterations guess with a real-time loop-gain (Aβ) monitor with five named threshold bands and best-so-far rollback.
|
|
5
5
|
Author-email: Dave Fitzsimmons <hello@loopgain.ai>
|
|
6
6
|
License: Apache-2.0
|
|
@@ -102,7 +102,6 @@ result = lg.result
|
|
|
102
102
|
print(result.outcome) # "converged" | "oscillating" | "diverged" | "stalled" | "max_iterations"
|
|
103
103
|
print(result.best_output) # the lowest-error iteration's output
|
|
104
104
|
print(result.iterations_used)
|
|
105
|
-
print(result.gain_margin) # 1 / max(Aβ_smooth)
|
|
106
105
|
print(result.savings_vs_fixed_cap)
|
|
107
106
|
```
|
|
108
107
|
|
|
@@ -183,6 +182,7 @@ LoopGain saves money by stopping a loop once it stops improving — fewer iterat
|
|
|
183
182
|
|
|
184
183
|
- **Savings depend on your workload.** Loops that usually succeed fast save the most (~96%); adversarial, failure-prone loops save less (~78–84%). The headline is a blend — run the benchmark on your own loops before quoting a number.
|
|
185
184
|
- **LoopGain detects convergence, not correctness.** It stops when your error signal stops improving — which means more iterations won't help, *not* that the loop succeeded. On the benchmark this preserved quality (it rarely stopped early on a worse output; false-stop rate ≤4.5%), but a loop can stall with the error still above zero — a plateau at, say, 2 failing tests. So check `result.best_error` (or your own pass/fail) before you trust the output: if it plateaued short of your target, that's a quality gap LoopGain can't see, and a false stop that forces a rerun is the one way it eats into the savings. LoopGain decides *when to stop*; you decide *whether the answer is good enough*.
|
|
185
|
+
- **LoopGain is only as right as your verifier.** It acts on the error signal you give it. If your verifier reports zero errors, LoopGain trusts that and stops — so a verifier with blind spots can report success on an answer that is still wrong, and LoopGain will confidently stop there. This is not the plateau case above: the error reads zero and the loop looks like a clean success, so neither LoopGain nor its convergence signal can flag it. The quality of the stop is bounded by the quality of the check behind your error signal. Pair LoopGain with the strongest verifier you can afford at the stop — executable tests over a sampled subset, a schema or type check over a vibe, a held-out check the loop didn't optimize against.
|
|
186
186
|
|
|
187
187
|
---
|
|
188
188
|
|
|
@@ -212,17 +212,9 @@ Returns `False` once a terminal state fires.
|
|
|
212
212
|
|
|
213
213
|
Current state name. One of `INIT`, `FAST_CONVERGE`, `CONVERGING`, `STALLING`, `OSCILLATING`, `DIVERGING`, `TARGET_MET`, `MAX_ITERATIONS`. The corresponding terminal `result.outcome` values are `converged`, `oscillating`, `diverged`, `stalled` (v0.2 trajectory mode only — STALLING terminating after 2 consecutive readings), `max_iterations`, or `in_progress`.
|
|
214
214
|
|
|
215
|
-
### `lg.eta -> int | None`
|
|
216
|
-
|
|
217
|
-
Best-effort closed-form estimate of iterations remaining, exposed for instrumentation. Returns `None` whenever it isn't well-defined — which is most of the time on real, jump-dominated loops, so don't depend on it for control.
|
|
218
|
-
|
|
219
|
-
### `lg.gain_margin -> float | None`
|
|
220
|
-
|
|
221
|
-
`1 / max(Aβ_smooth)`. `> 1` means stable headroom across the entire run.
|
|
222
|
-
|
|
223
215
|
### `lg.result -> LoopGainResult`
|
|
224
216
|
|
|
225
|
-
Terminal result with `outcome`, `iterations_used`, `best_index`, `best_output`, `best_error`, `convergence_profile`, `error_history`, `
|
|
217
|
+
Terminal result with `outcome`, `iterations_used`, `best_index`, `best_output`, `best_error`, `convergence_profile`, `error_history`, `savings_vs_fixed_cap`. Safe to call mid-loop.
|
|
226
218
|
|
|
227
219
|
### `lg.send_telemetry(endpoint, token, workload_id=None, timeout=2.0, allow_insecure=False, framework=None, loop_type=None, team=None, include_per_iteration=True) -> bool`
|
|
228
220
|
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
Covers the five band transitions, the three canonical scenarios from the
|
|
4
4
|
spec (converging / oscillating / diverging), TARGET_MET short-circuit,
|
|
5
|
-
best-so-far buffer correctness
|
|
5
|
+
and best-so-far buffer correctness.
|
|
6
6
|
"""
|
|
7
7
|
|
|
8
8
|
from __future__ import annotations
|
|
@@ -179,109 +179,6 @@ def test_best_so_far_works_without_outputs():
|
|
|
179
179
|
assert result.best_error == 2.0
|
|
180
180
|
|
|
181
181
|
|
|
182
|
-
# ----- ETA prediction calibration -----
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
def test_eta_prediction_calibration():
|
|
186
|
-
"""Synthetic monotonic decay: predicted iterations matches actual ±1."""
|
|
187
|
-
target = 0.1
|
|
188
|
-
ab = 0.5
|
|
189
|
-
errors = _decay(ab, e0=100.0)
|
|
190
|
-
# Iteration index where error first drops below target.
|
|
191
|
-
actual_n = next(i for i, e in enumerate(errors) if e < target)
|
|
192
|
-
|
|
193
|
-
lg = LoopGain(target_error=target, max_iterations=100)
|
|
194
|
-
# Feed a few errors so smoothed Aβ stabilizes.
|
|
195
|
-
lg.observe(errors[0])
|
|
196
|
-
lg.observe(errors[1])
|
|
197
|
-
lg.observe(errors[2])
|
|
198
|
-
# ETA at this point should predict ~(actual_n - 2) more iterations.
|
|
199
|
-
eta = lg.eta
|
|
200
|
-
assert eta is not None
|
|
201
|
-
assert abs(eta - (actual_n - 2)) <= 1, f"eta={eta}, expected ~{actual_n - 2}"
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
def test_eta_none_when_not_converging():
|
|
205
|
-
"""ETA is None when Aβ_smooth >= 1 (non-converging)."""
|
|
206
|
-
lg = LoopGain(target_error=0.5)
|
|
207
|
-
for _ in range(3):
|
|
208
|
-
lg.observe(10.0) # constant errors -> Aβ = 1
|
|
209
|
-
if not lg.should_continue():
|
|
210
|
-
break
|
|
211
|
-
assert lg.eta is None
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
def test_eta_none_when_target_is_zero():
|
|
215
|
-
lg = LoopGain(target_error=0.0)
|
|
216
|
-
lg.observe(100.0)
|
|
217
|
-
lg.observe(50.0)
|
|
218
|
-
assert lg.eta is None
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
# ----- First-eta snapshot (for ETA Accuracy dashboard panel) -----
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
def test_first_eta_snapshot_captured_during_converging_run():
|
|
225
|
-
"""Result carries the first non-None eta and the iter it was made at."""
|
|
226
|
-
target = 0.1
|
|
227
|
-
errors = _decay(0.5, e0=100.0)
|
|
228
|
-
lg = LoopGain(target_error=target, max_iterations=100)
|
|
229
|
-
for e in errors:
|
|
230
|
-
lg.observe(e)
|
|
231
|
-
if not lg.should_continue():
|
|
232
|
-
break
|
|
233
|
-
|
|
234
|
-
result = lg.result
|
|
235
|
-
# First eta becomes computable on the 2nd observation (smoothed_history
|
|
236
|
-
# exists, target > 0, current > target, Aβ_smooth < 1).
|
|
237
|
-
assert result.first_eta_at_iteration == 2
|
|
238
|
-
assert result.first_eta_prediction is not None
|
|
239
|
-
assert result.first_eta_prediction > 0
|
|
240
|
-
# Predicted total iterations should be within ±2 of actual (the prediction
|
|
241
|
-
# is made early, before smoothing has fully settled).
|
|
242
|
-
predicted_total = result.first_eta_at_iteration + result.first_eta_prediction
|
|
243
|
-
assert abs(predicted_total - result.iterations_used) <= 2
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
def test_first_eta_snapshot_none_when_target_is_zero():
|
|
247
|
-
"""No prediction is captured when target_error=0 (eta is always None)."""
|
|
248
|
-
lg = LoopGain(target_error=0.0, max_iterations=5)
|
|
249
|
-
for _ in range(5):
|
|
250
|
-
lg.observe(10.0)
|
|
251
|
-
result = lg.result
|
|
252
|
-
assert result.first_eta_prediction is None
|
|
253
|
-
assert result.first_eta_at_iteration is None
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
def test_first_eta_snapshot_none_when_loop_never_converges():
|
|
257
|
-
"""Oscillating loop (Aβ ≈ 1) never produces a positive eta."""
|
|
258
|
-
lg = LoopGain(target_error=0.5)
|
|
259
|
-
for _ in range(5):
|
|
260
|
-
lg.observe(10.0)
|
|
261
|
-
if not lg.should_continue():
|
|
262
|
-
break
|
|
263
|
-
result = lg.result
|
|
264
|
-
assert result.first_eta_prediction is None
|
|
265
|
-
assert result.first_eta_at_iteration is None
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
def test_first_eta_snapshot_is_idempotent():
|
|
269
|
-
"""Subsequent observations don't overwrite the first prediction."""
|
|
270
|
-
target = 0.1
|
|
271
|
-
errors = _decay(0.5, e0=100.0)
|
|
272
|
-
lg = LoopGain(target_error=target, max_iterations=100)
|
|
273
|
-
lg.observe(errors[0])
|
|
274
|
-
lg.observe(errors[1])
|
|
275
|
-
first = lg._first_eta_prediction
|
|
276
|
-
first_iter = lg._first_eta_at_iteration
|
|
277
|
-
assert first is not None
|
|
278
|
-
# Run a few more iterations; the snapshot should not change.
|
|
279
|
-
for e in errors[2:6]:
|
|
280
|
-
lg.observe(e)
|
|
281
|
-
assert lg._first_eta_prediction == first
|
|
282
|
-
assert lg._first_eta_at_iteration == first_iter
|
|
283
|
-
|
|
284
|
-
|
|
285
182
|
# ----- observe() input coercion -----
|
|
286
183
|
|
|
287
184
|
|
|
@@ -353,23 +250,6 @@ def test_max_iterations_with_converging_loop():
|
|
|
353
250
|
assert not lg.should_continue()
|
|
354
251
|
|
|
355
252
|
|
|
356
|
-
# ----- gain_margin -----
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
def test_gain_margin_greater_than_one_for_converging():
|
|
360
|
-
lg = LoopGain(max_iterations=10)
|
|
361
|
-
for e in _decay(0.5)[:5]:
|
|
362
|
-
lg.observe(e)
|
|
363
|
-
gm = lg.gain_margin
|
|
364
|
-
assert gm is not None
|
|
365
|
-
assert gm > 1.0
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
def test_gain_margin_none_before_observations():
|
|
369
|
-
lg = LoopGain()
|
|
370
|
-
assert lg.gain_margin is None
|
|
371
|
-
|
|
372
|
-
|
|
373
253
|
# ----- result before any observations -----
|
|
374
254
|
|
|
375
255
|
|
|
@@ -130,8 +130,7 @@ def test_very_large_error_magnitudes():
|
|
|
130
130
|
lg.observe(e)
|
|
131
131
|
# Aβ = 0.4 → FAST_CONVERGE.
|
|
132
132
|
assert lg.state in (FAST_CONVERGE, CONVERGING)
|
|
133
|
-
assert lg.
|
|
134
|
-
assert lg.gain_margin > 1.0
|
|
133
|
+
assert lg.result.convergence_profile # Aβ computed without overflow
|
|
135
134
|
|
|
136
135
|
|
|
137
136
|
def test_zero_error_in_middle_of_run_anomalous():
|
|
@@ -245,19 +244,6 @@ def test_result_callable_mid_loop():
|
|
|
245
244
|
assert r2.iterations_used == 3
|
|
246
245
|
|
|
247
246
|
|
|
248
|
-
def test_eta_before_any_observation():
|
|
249
|
-
"""eta on a fresh instance is None, not a crash."""
|
|
250
|
-
lg = LoopGain(target_error=0.5)
|
|
251
|
-
assert lg.eta is None
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
def test_gain_margin_with_single_observation():
|
|
255
|
-
"""Single observation has no Aβ yet → gain_margin is None."""
|
|
256
|
-
lg = LoopGain()
|
|
257
|
-
lg.observe(10.0)
|
|
258
|
-
assert lg.gain_margin is None
|
|
259
|
-
|
|
260
|
-
|
|
261
247
|
# ----- Mixed input types -----
|
|
262
248
|
|
|
263
249
|
|
|
@@ -53,7 +53,6 @@ def test_payload_loop_section_has_outcome_and_stats():
|
|
|
53
53
|
loop = p["loop"]
|
|
54
54
|
assert loop["outcome"] == "converged"
|
|
55
55
|
assert loop["iterations_used"] == 4
|
|
56
|
-
assert loop["gain_margin"] is not None
|
|
57
56
|
assert loop["savings_vs_fixed_cap"] is not None
|
|
58
57
|
assert "convergence_profile_summary" in loop
|
|
59
58
|
assert "rollback_triggered" in loop
|
|
@@ -120,7 +119,7 @@ def test_payload_workload_id_optional():
|
|
|
120
119
|
|
|
121
120
|
|
|
122
121
|
def test_payload_serializes_strict_json_for_constant_error_trajectory():
|
|
123
|
-
"""A
|
|
122
|
+
"""A zero-error trajectory pushes Aβ to +inf (E(n)/E(n-1) with E(n-1)=0).
|
|
124
123
|
|
|
125
124
|
Standard JSON forbids Infinity / NaN, and the receiver rejects payloads
|
|
126
125
|
that include them. The build_payload sanitizer must coerce non-finite
|
|
@@ -135,8 +134,8 @@ def test_payload_serializes_strict_json_for_constant_error_trajectory():
|
|
|
135
134
|
# Strict round-trip: allow_nan=False raises on inf/nan.
|
|
136
135
|
encoded = json.dumps(p, allow_nan=False)
|
|
137
136
|
decoded = json.loads(encoded)
|
|
138
|
-
|
|
139
|
-
#
|
|
137
|
+
# Per-iteration Aβ values can be non-finite (E(n)/E(n-1) with E(n-1)=0)
|
|
138
|
+
# and the convergence-profile summary must stay finite-or-None too.
|
|
140
139
|
for v in decoded["per_iteration"]["convergence_profile"]:
|
|
141
140
|
assert v is None or isinstance(v, (int, float))
|
|
142
141
|
|
|
@@ -190,50 +189,24 @@ def test_payload_for_not_started_loop():
|
|
|
190
189
|
assert p["loop"]["convergence_profile_summary"]["samples"] == 0
|
|
191
190
|
|
|
192
191
|
|
|
193
|
-
# -----
|
|
192
|
+
# ----- schema version -----
|
|
194
193
|
|
|
195
194
|
|
|
196
|
-
def
|
|
197
|
-
"""Schema bumped to
|
|
198
|
-
assert SCHEMA_VERSION ==
|
|
195
|
+
def test_payload_schema_version_is_v4():
|
|
196
|
+
"""Schema bumped to v4 when ETA + gain_margin were removed from the payload."""
|
|
197
|
+
assert SCHEMA_VERSION == 4
|
|
199
198
|
lg = _make_terminated_loop()
|
|
200
199
|
p = build_payload(lg)
|
|
201
|
-
assert p["schema_version"] ==
|
|
200
|
+
assert p["schema_version"] == 4
|
|
202
201
|
|
|
203
202
|
|
|
204
|
-
def
|
|
205
|
-
"""
|
|
203
|
+
def test_payload_loop_section_drops_eta_and_gain_margin():
|
|
204
|
+
"""v4 no longer carries the discontinued ETA / gain_margin fields."""
|
|
206
205
|
lg = _make_terminated_loop()
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
assert "first_eta_prediction" in loop
|
|
210
|
-
assert "first_eta_at_iteration" in loop
|
|
211
|
-
assert loop["first_eta_prediction"] is not None
|
|
212
|
-
assert loop["first_eta_at_iteration"] is not None
|
|
213
|
-
assert loop["first_eta_prediction"] > 0
|
|
214
|
-
assert loop["first_eta_at_iteration"] >= 2
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
def test_payload_first_eta_none_for_target_zero():
|
|
218
|
-
"""target_error=0 means eta is never computable; both fields are None."""
|
|
219
|
-
lg = LoopGain(target_error=0.0, max_iterations=4)
|
|
220
|
-
for _ in range(4):
|
|
221
|
-
lg.observe(10.0)
|
|
222
|
-
p = build_payload(lg)
|
|
223
|
-
assert p["loop"]["first_eta_prediction"] is None
|
|
224
|
-
assert p["loop"]["first_eta_at_iteration"] is None
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
def test_payload_first_eta_none_for_diverging_loop():
|
|
228
|
-
"""A divergent loop never produces a positive eta."""
|
|
229
|
-
lg = LoopGain(target_error=0.5, max_iterations=20)
|
|
230
|
-
for e in [10.0, 12.0, 15.0, 20.0, 30.0]:
|
|
231
|
-
if not lg.should_continue():
|
|
232
|
-
break
|
|
233
|
-
lg.observe(e)
|
|
234
|
-
p = build_payload(lg)
|
|
235
|
-
assert p["loop"]["first_eta_prediction"] is None
|
|
236
|
-
assert p["loop"]["first_eta_at_iteration"] is None
|
|
206
|
+
loop = build_payload(lg)["loop"]
|
|
207
|
+
assert "gain_margin" not in loop
|
|
208
|
+
assert "first_eta_prediction" not in loop
|
|
209
|
+
assert "first_eta_at_iteration" not in loop
|
|
237
210
|
|
|
238
211
|
|
|
239
212
|
# ----- send_payload behavior -----
|
|
@@ -660,3 +633,117 @@ def test_send_payload_refuses_redirects():
|
|
|
660
633
|
req = urllib.request.Request("https://example.com/")
|
|
661
634
|
with pytest.raises(urllib.error.HTTPError):
|
|
662
635
|
method(req, io.BytesIO(b""), 302, "Found", {})
|
|
636
|
+
|
|
637
|
+
|
|
638
|
+
# ----- send_payload retry behavior (transient failures) -----
|
|
639
|
+
|
|
640
|
+
import socket as _socket
|
|
641
|
+
import urllib.error as _uerr
|
|
642
|
+
|
|
643
|
+
from loopgain import telemetry as _tele
|
|
644
|
+
|
|
645
|
+
|
|
646
|
+
class _OkResp:
|
|
647
|
+
status = 202
|
|
648
|
+
|
|
649
|
+
def __enter__(self):
|
|
650
|
+
return self
|
|
651
|
+
|
|
652
|
+
def __exit__(self, *args):
|
|
653
|
+
pass
|
|
654
|
+
|
|
655
|
+
|
|
656
|
+
def _retry_payload():
|
|
657
|
+
return build_payload(_make_terminated_loop(), workload_id="retry-test")
|
|
658
|
+
|
|
659
|
+
|
|
660
|
+
def test_send_payload_retries_transient_then_succeeds(monkeypatch):
|
|
661
|
+
"""A transient failure (timeout) is retried; a later success returns True."""
|
|
662
|
+
calls = {"n": 0}
|
|
663
|
+
|
|
664
|
+
def flaky(req, timeout=None):
|
|
665
|
+
calls["n"] += 1
|
|
666
|
+
if calls["n"] < 3:
|
|
667
|
+
raise _socket.timeout("slow first attempts")
|
|
668
|
+
return _OkResp()
|
|
669
|
+
|
|
670
|
+
sleeps: list[float] = []
|
|
671
|
+
monkeypatch.setattr("loopgain.telemetry._open_request", flaky)
|
|
672
|
+
monkeypatch.setattr("loopgain.telemetry.time.sleep", lambda s: sleeps.append(s))
|
|
673
|
+
|
|
674
|
+
ok = send_payload("https://t.example/v1/aggregate", token="t", payload=_retry_payload())
|
|
675
|
+
assert ok is True
|
|
676
|
+
assert calls["n"] == 3 # two transient failures, third succeeds
|
|
677
|
+
assert sleeps == [0.25, 0.5] # linear backoff between attempts
|
|
678
|
+
|
|
679
|
+
|
|
680
|
+
def test_send_payload_gives_up_after_retries_on_persistent_5xx(monkeypatch):
|
|
681
|
+
"""A persistent transient (503) exhausts retries and returns False."""
|
|
682
|
+
calls = {"n": 0}
|
|
683
|
+
|
|
684
|
+
def always_503(req, timeout=None):
|
|
685
|
+
calls["n"] += 1
|
|
686
|
+
raise _uerr.HTTPError("https://t.example", 503, "unavailable", {}, None)
|
|
687
|
+
|
|
688
|
+
monkeypatch.setattr("loopgain.telemetry._open_request", always_503)
|
|
689
|
+
monkeypatch.setattr("loopgain.telemetry.time.sleep", lambda s: None)
|
|
690
|
+
|
|
691
|
+
ok = send_payload("https://t.example/v1/aggregate", token="t", payload=_retry_payload(), retries=2)
|
|
692
|
+
assert ok is False
|
|
693
|
+
assert calls["n"] == 3 # 1 initial + 2 retries
|
|
694
|
+
|
|
695
|
+
|
|
696
|
+
def test_send_payload_does_not_retry_deterministic_4xx(monkeypatch):
|
|
697
|
+
"""A 401 will never succeed on retry — fail fast, no backoff."""
|
|
698
|
+
calls = {"n": 0}
|
|
699
|
+
slept = {"n": 0}
|
|
700
|
+
|
|
701
|
+
def unauthorized(req, timeout=None):
|
|
702
|
+
calls["n"] += 1
|
|
703
|
+
raise _uerr.HTTPError("https://t.example", 401, "unauthorized", {}, None)
|
|
704
|
+
|
|
705
|
+
monkeypatch.setattr("loopgain.telemetry._open_request", unauthorized)
|
|
706
|
+
monkeypatch.setattr("loopgain.telemetry.time.sleep", lambda s: slept.__setitem__("n", slept["n"] + 1))
|
|
707
|
+
|
|
708
|
+
ok = send_payload("https://t.example/v1/aggregate", token="bad", payload=_retry_payload())
|
|
709
|
+
assert ok is False
|
|
710
|
+
assert calls["n"] == 1 # no retry on a deterministic 4xx
|
|
711
|
+
assert slept["n"] == 0
|
|
712
|
+
|
|
713
|
+
|
|
714
|
+
def test_send_payload_retries_zero_is_single_shot(monkeypatch):
|
|
715
|
+
"""retries=0 restores the original single-attempt behavior."""
|
|
716
|
+
calls = {"n": 0}
|
|
717
|
+
|
|
718
|
+
def timeout(req, timeout=None):
|
|
719
|
+
calls["n"] += 1
|
|
720
|
+
raise TimeoutError()
|
|
721
|
+
|
|
722
|
+
monkeypatch.setattr("loopgain.telemetry._open_request", timeout)
|
|
723
|
+
monkeypatch.setattr("loopgain.telemetry.time.sleep", lambda s: None)
|
|
724
|
+
|
|
725
|
+
ok = send_payload("https://t.example/v1/aggregate", token="t", payload=_retry_payload(), retries=0)
|
|
726
|
+
assert ok is False
|
|
727
|
+
assert calls["n"] == 1
|
|
728
|
+
|
|
729
|
+
|
|
730
|
+
def test_send_payload_never_raises_on_unexpected_error(monkeypatch):
|
|
731
|
+
"""A non-transient, unexpected error is swallowed (best-effort), no retry."""
|
|
732
|
+
def boom(req, timeout=None):
|
|
733
|
+
raise RuntimeError("unexpected")
|
|
734
|
+
|
|
735
|
+
monkeypatch.setattr("loopgain.telemetry._open_request", boom)
|
|
736
|
+
monkeypatch.setattr("loopgain.telemetry.time.sleep", lambda s: None)
|
|
737
|
+
|
|
738
|
+
assert send_payload("https://t.example/v1/aggregate", token="t", payload=_retry_payload()) is False
|
|
739
|
+
|
|
740
|
+
|
|
741
|
+
def test_is_transient_classification():
|
|
742
|
+
assert _tele._is_transient(TimeoutError()) is True
|
|
743
|
+
assert _tele._is_transient(_socket.timeout()) is True
|
|
744
|
+
assert _tele._is_transient(_uerr.URLError("dns")) is True
|
|
745
|
+
assert _tele._is_transient(_uerr.HTTPError("u", 503, "x", {}, None)) is True
|
|
746
|
+
assert _tele._is_transient(_uerr.HTTPError("u", 429, "x", {}, None)) is True
|
|
747
|
+
assert _tele._is_transient(_uerr.HTTPError("u", 400, "x", {}, None)) is False
|
|
748
|
+
assert _tele._is_transient(_uerr.HTTPError("u", 401, "x", {}, None)) is False
|
|
749
|
+
assert _tele._is_transient(RuntimeError("x")) is False
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|