evenkeel 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- evenkeel/__init__.py +34 -0
- evenkeel/_compat.py +21 -0
- evenkeel/_core.py +126 -0
- evenkeel/budget.py +74 -0
- evenkeel/failure.py +192 -0
- evenkeel/jev.py +170 -0
- evenkeel/middleware.py +399 -0
- evenkeel-0.1.0.dist-info/METADATA +144 -0
- evenkeel-0.1.0.dist-info/RECORD +11 -0
- evenkeel-0.1.0.dist-info/WHEEL +4 -0
- evenkeel-0.1.0.dist-info/licenses/LICENSE +21 -0
evenkeel/__init__.py
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
"""Failure classification, retries with hints, dollar budgets and escalation for LangChain agents."""
|
|
2
|
+
|
|
3
|
+
from ._core import AttemptRecord
|
|
4
|
+
from .budget import ModelPrice, PriceTable
|
|
5
|
+
from .failure import (
|
|
6
|
+
BadInputError,
|
|
7
|
+
BadOutputError,
|
|
8
|
+
FailureContext,
|
|
9
|
+
FailureVerdict,
|
|
10
|
+
NeedsHumanError,
|
|
11
|
+
OverBudgetError,
|
|
12
|
+
RuleClassifier,
|
|
13
|
+
default_policy,
|
|
14
|
+
)
|
|
15
|
+
from .jev import JevClassifier
|
|
16
|
+
from .middleware import EvenKeelMiddleware
|
|
17
|
+
|
|
18
|
+
__version__ = "0.1.0"
|
|
19
|
+
|
|
20
|
+
__all__ = [
|
|
21
|
+
"AttemptRecord",
|
|
22
|
+
"BadInputError",
|
|
23
|
+
"BadOutputError",
|
|
24
|
+
"EvenKeelMiddleware",
|
|
25
|
+
"FailureContext",
|
|
26
|
+
"FailureVerdict",
|
|
27
|
+
"JevClassifier",
|
|
28
|
+
"ModelPrice",
|
|
29
|
+
"NeedsHumanError",
|
|
30
|
+
"OverBudgetError",
|
|
31
|
+
"PriceTable",
|
|
32
|
+
"RuleClassifier",
|
|
33
|
+
"default_policy",
|
|
34
|
+
]
|
evenkeel/_compat.py
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
"""The one place that imports LangChain and LangGraph internals. If an import path moves in a new release,
|
|
2
|
+
change it here only."""
|
|
3
|
+
|
|
4
|
+
from langchain.agents.middleware import AgentMiddleware, AgentState, ModelRequest, ModelResponse, hook_config
|
|
5
|
+
from langchain.agents.middleware.types import ExtendedModelResponse
|
|
6
|
+
from langgraph.errors import GraphBubbleUp
|
|
7
|
+
from langgraph.prebuilt.tool_node import ToolCallRequest
|
|
8
|
+
from langgraph.types import Command, interrupt
|
|
9
|
+
|
|
10
|
+
__all__ = [
|
|
11
|
+
"AgentMiddleware",
|
|
12
|
+
"AgentState",
|
|
13
|
+
"Command",
|
|
14
|
+
"ExtendedModelResponse",
|
|
15
|
+
"GraphBubbleUp",
|
|
16
|
+
"ModelRequest",
|
|
17
|
+
"ModelResponse",
|
|
18
|
+
"ToolCallRequest",
|
|
19
|
+
"hook_config",
|
|
20
|
+
"interrupt",
|
|
21
|
+
]
|
evenkeel/_core.py
ADDED
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
"""Decision logic shared by every integration: apply the policy, enforce the attempt limit, build records,
|
|
2
|
+
and read reviewers' resume values. No LangChain imports, so other integrations can reuse it."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
from dataclasses import dataclass
|
|
7
|
+
from typing import Any, Literal
|
|
8
|
+
|
|
9
|
+
from typing_extensions import TypedDict
|
|
10
|
+
|
|
11
|
+
from .failure import (
|
|
12
|
+
Escalate,
|
|
13
|
+
Fail,
|
|
14
|
+
FailureAction,
|
|
15
|
+
FailureContext,
|
|
16
|
+
FailurePolicy,
|
|
17
|
+
FailureVerdict,
|
|
18
|
+
Retry,
|
|
19
|
+
RetryWithHint,
|
|
20
|
+
error_message,
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
_RECORD_ERROR_CHARS = 500
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class AttemptRecord(TypedDict):
|
|
27
|
+
attempt: int
|
|
28
|
+
call: str
|
|
29
|
+
error_type: str
|
|
30
|
+
error: str
|
|
31
|
+
kind: str
|
|
32
|
+
confidence: float
|
|
33
|
+
source: str
|
|
34
|
+
action: str
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
@dataclass(frozen=True)
|
|
38
|
+
class Decision:
|
|
39
|
+
action: FailureAction
|
|
40
|
+
record: AttemptRecord
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def decide(ctx: FailureContext, verdict: FailureVerdict, policy: FailurePolicy) -> Decision:
|
|
44
|
+
action = policy(verdict, ctx)
|
|
45
|
+
if isinstance(action, (Retry, RetryWithHint, Escalate)) and ctx.attempt >= ctx.max_attempts:
|
|
46
|
+
action = Fail(reason=f"attempt limit reached ({ctx.max_attempts}): {error_message(ctx.error)}")
|
|
47
|
+
record = AttemptRecord(
|
|
48
|
+
attempt=ctx.attempt,
|
|
49
|
+
call=ctx.step or "",
|
|
50
|
+
error_type=type(ctx.error).__name__,
|
|
51
|
+
error=error_message(ctx.error)[:_RECORD_ERROR_CHARS],
|
|
52
|
+
kind=verdict.kind,
|
|
53
|
+
confidence=round(verdict.confidence, 3),
|
|
54
|
+
source=verdict.source,
|
|
55
|
+
action=action.type,
|
|
56
|
+
)
|
|
57
|
+
return Decision(action=action, record=record)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def escalation_payload(ctx: FailureContext, verdict: FailureVerdict, reason: str) -> dict[str, Any]:
|
|
61
|
+
is_tool = (ctx.step or "").startswith("tool:")
|
|
62
|
+
return {
|
|
63
|
+
"type": "evenkeel_escalation",
|
|
64
|
+
"call": ctx.step or "",
|
|
65
|
+
"error_type": type(ctx.error).__name__,
|
|
66
|
+
"error": error_message(ctx.error)[:_RECORD_ERROR_CHARS],
|
|
67
|
+
"kind": verdict.kind,
|
|
68
|
+
"confidence": round(verdict.confidence, 3),
|
|
69
|
+
"reason": reason,
|
|
70
|
+
"accepted": ["retry", "fail", "skip"] if is_tool else ["retry", "fail"],
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
@dataclass(frozen=True)
|
|
75
|
+
class Resume:
|
|
76
|
+
decision: Literal["retry", "fail", "skip"]
|
|
77
|
+
hint: str | None
|
|
78
|
+
message: str | None
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
_RESUME_HELP = (
|
|
82
|
+
'Resume with {"decision": "retry"} (optionally with "hint": "..."), {"decision": "fail"}, '
|
|
83
|
+
'or, for tool calls only, {"decision": "skip", "message": "..."}'
|
|
84
|
+
)
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def parse_resume(value: Any, *, call_type: Literal["model", "tool"]) -> Resume:
|
|
88
|
+
if not isinstance(value, dict) or value.get("decision") not in ("retry", "fail", "skip"):
|
|
89
|
+
raise ValueError(f"Unrecognised escalation decision {value!r}. {_RESUME_HELP}")
|
|
90
|
+
decision = value["decision"]
|
|
91
|
+
hint = value.get("hint")
|
|
92
|
+
message = value.get("message")
|
|
93
|
+
if hint is not None and not isinstance(hint, str):
|
|
94
|
+
raise ValueError(f"The decision's hint must be a string, got {hint!r}. {_RESUME_HELP}")
|
|
95
|
+
if decision == "skip":
|
|
96
|
+
if call_type == "model":
|
|
97
|
+
raise ValueError(f"A model call cannot be skipped: it has no stand-in result. {_RESUME_HELP}")
|
|
98
|
+
if not isinstance(message, str):
|
|
99
|
+
raise ValueError(f"A skip decision needs a message for the model. {_RESUME_HELP}")
|
|
100
|
+
return Resume("skip", None, message)
|
|
101
|
+
return Resume(decision, hint, None)
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
_BUDGET_HELP = 'Resume with {"approve": true, "budget_usd": <more than spent>} or {"approve": false}'
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def parse_budget_resume(value: Any, *, spent_usd: float) -> float | None:
|
|
108
|
+
if not isinstance(value, dict) or not isinstance(value.get("approve"), bool):
|
|
109
|
+
raise ValueError(f"Unrecognised budget decision {value!r}. {_BUDGET_HELP}")
|
|
110
|
+
if not value["approve"]:
|
|
111
|
+
return None
|
|
112
|
+
budget = value.get("budget_usd")
|
|
113
|
+
if not isinstance(budget, (int, float)) or isinstance(budget, bool) or budget <= spent_usd:
|
|
114
|
+
raise ValueError(
|
|
115
|
+
f"An approval needs a budget_usd above the ${spent_usd:.4f} already spent, "
|
|
116
|
+
f"got {budget!r}. {_BUDGET_HELP}"
|
|
117
|
+
)
|
|
118
|
+
return float(budget)
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def attach_attempts(error: BaseException, records: list[AttemptRecord]) -> None:
|
|
122
|
+
try:
|
|
123
|
+
error.evenkeel_attempts = list(records) # type: ignore[attr-defined]
|
|
124
|
+
except AttributeError: # some built-in exceptions refuse new attributes
|
|
125
|
+
if hasattr(error, "add_note"): # Python 3.11+
|
|
126
|
+
error.add_note(f"evenkeel attempts: {records}")
|
evenkeel/budget.py
ADDED
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
"""Turns a model response's token usage into dollars, from prices the user supplies."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
from collections.abc import Mapping
|
|
7
|
+
from dataclasses import dataclass
|
|
8
|
+
|
|
9
|
+
from langchain_core.messages import AIMessage
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class UnknownModelPriceError(ValueError):
|
|
13
|
+
"""The response's model is not in the price table."""
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class MissingUsageError(ValueError):
|
|
17
|
+
"""The response reported no token usage, so its cost is unknown."""
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@dataclass(frozen=True)
|
|
21
|
+
class ModelPrice:
|
|
22
|
+
input_per_mtok: float
|
|
23
|
+
"""Dollars per million input tokens."""
|
|
24
|
+
output_per_mtok: float
|
|
25
|
+
"""Dollars per million output tokens."""
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def model_name_of(message: AIMessage) -> str | None:
|
|
29
|
+
metadata = message.response_metadata or {}
|
|
30
|
+
name = metadata.get("model_name") or metadata.get("model")
|
|
31
|
+
return name if isinstance(name, str) else None
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
# What may follow a priced name: only a date or version number (-2024-07-18, -20241022, -0613, -001) or the
|
|
35
|
+
# -latest alias. Anything else (-5-sonnet, -turbo, -1106-preview) is a different model with its own price.
|
|
36
|
+
_VERSION_SUFFIX = re.compile(r"-(\d{4}-\d{2}-\d{2}|\d{3,8}|latest)")
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class PriceTable:
|
|
40
|
+
"""Prices per model name. A name with a version or date suffix uses its base price
|
|
41
|
+
(gpt-4o-mini-2024-07-18 -> gpt-4o-mini); a different model never borrows another's price
|
|
42
|
+
(gpt-4o-mini does not use gpt-4, and gpt-4-turbo does not use gpt-4)."""
|
|
43
|
+
|
|
44
|
+
def __init__(self, prices: Mapping[str, ModelPrice]) -> None:
|
|
45
|
+
self._prices = dict(prices)
|
|
46
|
+
|
|
47
|
+
def price_for(self, model_name: str) -> ModelPrice:
|
|
48
|
+
if model_name in self._prices:
|
|
49
|
+
return self._prices[model_name]
|
|
50
|
+
matches = [
|
|
51
|
+
key
|
|
52
|
+
for key in self._prices
|
|
53
|
+
if model_name.startswith(key) and _VERSION_SUFFIX.fullmatch(model_name[len(key) :])
|
|
54
|
+
]
|
|
55
|
+
if not matches:
|
|
56
|
+
raise UnknownModelPriceError(
|
|
57
|
+
f"No price for model {model_name!r}. Add it to PriceTable; known: {sorted(self._prices)}"
|
|
58
|
+
)
|
|
59
|
+
return self._prices[max(matches, key=len)]
|
|
60
|
+
|
|
61
|
+
def cost_usd(self, message: AIMessage) -> float:
|
|
62
|
+
name = model_name_of(message)
|
|
63
|
+
if name is None:
|
|
64
|
+
raise UnknownModelPriceError(
|
|
65
|
+
"The response does not say which model produced it; response_metadata keys: "
|
|
66
|
+
f"{sorted((message.response_metadata or {}).keys())}"
|
|
67
|
+
)
|
|
68
|
+
price = self.price_for(name)
|
|
69
|
+
usage = message.usage_metadata
|
|
70
|
+
if not usage:
|
|
71
|
+
raise MissingUsageError(f"Model {name!r} reported no token usage, so this call's cost is unknown")
|
|
72
|
+
return (
|
|
73
|
+
usage["input_tokens"] * price.input_per_mtok + usage["output_tokens"] * price.output_per_mtok
|
|
74
|
+
) / 1e6
|
evenkeel/failure.py
ADDED
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
"""Failure kinds, the rule classifier and the default policy. Ported from keel's src/failure.ts."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import errno as _errno
|
|
6
|
+
import random as _random
|
|
7
|
+
import socket as _socket
|
|
8
|
+
from collections.abc import Callable
|
|
9
|
+
from dataclasses import dataclass, field
|
|
10
|
+
from typing import Any, Literal, Protocol
|
|
11
|
+
|
|
12
|
+
FailureKind = Literal["transient", "bad_input", "bad_output", "needs_human", "over_budget", "fatal"]
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@dataclass(frozen=True)
|
|
16
|
+
class FailureContext:
|
|
17
|
+
error: BaseException
|
|
18
|
+
task: str
|
|
19
|
+
attempt: int
|
|
20
|
+
"""The attempt that just failed, starting at 1."""
|
|
21
|
+
max_attempts: int
|
|
22
|
+
step: str | None = None
|
|
23
|
+
"""What was running: "model" or "tool:<name>"."""
|
|
24
|
+
payload: Any = None
|
|
25
|
+
output: Any = None
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@dataclass(frozen=True)
|
|
29
|
+
class FailureVerdict:
|
|
30
|
+
kind: FailureKind
|
|
31
|
+
confidence: float
|
|
32
|
+
"""0 to 1. Rule matches on explicit signals are near 1, fallbacks are low."""
|
|
33
|
+
source: str = "rules"
|
|
34
|
+
"""Which classifier decided, e.g. "rules", "jev", "rules (jev unavailable)"."""
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class Classifier(Protocol):
|
|
38
|
+
def classify(self, ctx: FailureContext) -> FailureVerdict: ...
|
|
39
|
+
|
|
40
|
+
async def aclassify(self, ctx: FailureContext) -> FailureVerdict: ...
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class EvenKeelError(Exception):
|
|
44
|
+
"""Base for explicit failure signals. `output` is what the model or tool produced."""
|
|
45
|
+
|
|
46
|
+
def __init__(self, message: str, *, output: Any = None) -> None:
|
|
47
|
+
super().__init__(message)
|
|
48
|
+
self.output = output
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
class BadInputError(EvenKeelError):
|
|
52
|
+
"""The input can never succeed, whatever the retry."""
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
class BadOutputError(EvenKeelError):
|
|
56
|
+
"""A model or tool produced unusable output (hallucinated tool, malformed JSON)."""
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class NeedsHumanError(EvenKeelError):
|
|
60
|
+
"""A person has to decide before the run can continue."""
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
class OverBudgetError(EvenKeelError):
|
|
64
|
+
"""The run has spent its budget."""
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
_TRANSIENT_NAMES = frozenset(
|
|
68
|
+
{
|
|
69
|
+
"APITimeoutError",
|
|
70
|
+
"APIConnectionError",
|
|
71
|
+
"ConnectTimeout",
|
|
72
|
+
"ReadTimeout",
|
|
73
|
+
"WriteTimeout",
|
|
74
|
+
"PoolTimeout",
|
|
75
|
+
"TimeoutException",
|
|
76
|
+
"ConnectError",
|
|
77
|
+
"ReadError",
|
|
78
|
+
"RemoteProtocolError",
|
|
79
|
+
}
|
|
80
|
+
)
|
|
81
|
+
_VALIDATION_NAMES = frozenset({"ValidationError"})
|
|
82
|
+
# Network error codes, as strings on `code` (keel's list) or as numbers on `errno` (OSError, socket.gaierror).
|
|
83
|
+
_TRANSIENT_CODES = frozenset(
|
|
84
|
+
{"ETIMEDOUT", "ECONNRESET", "ECONNREFUSED", "EPIPE", "EAI_AGAIN", "UND_ERR_SOCKET"}
|
|
85
|
+
)
|
|
86
|
+
_TRANSIENT_ERRNOS = frozenset({_errno.ETIMEDOUT, _errno.ECONNRESET, _errno.ECONNREFUSED, _errno.EPIPE})
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def error_message(error: BaseException) -> str:
|
|
90
|
+
return str(error) or type(error).__name__
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def _http_status(error: BaseException) -> int | None:
|
|
94
|
+
for attr in ("status_code", "status"):
|
|
95
|
+
value = getattr(error, attr, None)
|
|
96
|
+
if isinstance(value, int) and not isinstance(value, bool):
|
|
97
|
+
return value
|
|
98
|
+
value = getattr(getattr(error, "response", None), "status_code", None)
|
|
99
|
+
return value if isinstance(value, int) and not isinstance(value, bool) else None
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _is_network_error(error: BaseException) -> bool:
|
|
103
|
+
if isinstance(error, (TimeoutError, ConnectionError)) or type(error).__name__ in _TRANSIENT_NAMES:
|
|
104
|
+
return True
|
|
105
|
+
code, number = getattr(error, "code", None), getattr(error, "errno", None)
|
|
106
|
+
if isinstance(error, _socket.gaierror):
|
|
107
|
+
# Only DNS errors use EAI_* numbers; on macOS EAI_AGAIN equals errno.ENOENT (a missing file).
|
|
108
|
+
return number == _socket.EAI_AGAIN
|
|
109
|
+
return (isinstance(code, str) and code in _TRANSIENT_CODES) or number in _TRANSIENT_ERRNOS
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
class RuleClassifier:
|
|
113
|
+
"""Classifies failures from explicit signals only: error classes, HTTP status, network errors."""
|
|
114
|
+
|
|
115
|
+
def classify(self, ctx: FailureContext) -> FailureVerdict:
|
|
116
|
+
error = ctx.error
|
|
117
|
+
if isinstance(error, BadInputError):
|
|
118
|
+
return FailureVerdict("bad_input", 1.0)
|
|
119
|
+
if isinstance(error, BadOutputError):
|
|
120
|
+
return FailureVerdict("bad_output", 1.0)
|
|
121
|
+
if isinstance(error, NeedsHumanError):
|
|
122
|
+
return FailureVerdict("needs_human", 1.0)
|
|
123
|
+
if isinstance(error, OverBudgetError):
|
|
124
|
+
return FailureVerdict("over_budget", 1.0)
|
|
125
|
+
|
|
126
|
+
status = _http_status(error)
|
|
127
|
+
if status is not None:
|
|
128
|
+
if status in (408, 429) or status >= 500:
|
|
129
|
+
return FailureVerdict("transient", 0.9)
|
|
130
|
+
if status in (400, 422):
|
|
131
|
+
return FailureVerdict("bad_input", 0.8)
|
|
132
|
+
return FailureVerdict("fatal", 0.8)
|
|
133
|
+
if _is_network_error(error):
|
|
134
|
+
return FailureVerdict("transient", 0.9)
|
|
135
|
+
# SDKs often wrap the network error: look one level down the chain.
|
|
136
|
+
cause = error.__cause__ or error.__context__
|
|
137
|
+
if cause is not None and _is_network_error(cause):
|
|
138
|
+
return FailureVerdict("transient", 0.9)
|
|
139
|
+
if type(error).__name__ in _VALIDATION_NAMES:
|
|
140
|
+
return FailureVerdict("bad_input", 0.7)
|
|
141
|
+
return FailureVerdict("fatal", 0.5)
|
|
142
|
+
|
|
143
|
+
async def aclassify(self, ctx: FailureContext) -> FailureVerdict:
|
|
144
|
+
return self.classify(ctx)
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
@dataclass(frozen=True)
|
|
148
|
+
class Retry:
|
|
149
|
+
delay_s: float
|
|
150
|
+
type: Literal["retry"] = field(default="retry", init=False)
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
@dataclass(frozen=True)
|
|
154
|
+
class RetryWithHint:
|
|
155
|
+
hint: str
|
|
156
|
+
type: Literal["retry_modified"] = field(default="retry_modified", init=False)
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
@dataclass(frozen=True)
|
|
160
|
+
class Escalate:
|
|
161
|
+
reason: str
|
|
162
|
+
type: Literal["escalate"] = field(default="escalate", init=False)
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
@dataclass(frozen=True)
|
|
166
|
+
class Fail:
|
|
167
|
+
reason: str
|
|
168
|
+
type: Literal["fail"] = field(default="fail", init=False)
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
FailureAction = Retry | RetryWithHint | Escalate | Fail
|
|
172
|
+
FailurePolicy = Callable[[FailureVerdict, FailureContext], FailureAction]
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def default_policy(
|
|
176
|
+
*, base_s: float = 1.0, max_s: float = 60.0, random: Callable[[], float] = _random.random
|
|
177
|
+
) -> FailurePolicy:
|
|
178
|
+
"""Transient: exponential backoff with full jitter. Bad output: retry with the error as a hint.
|
|
179
|
+
Needs-human and over-budget: escalate. Bad input and fatal: fail."""
|
|
180
|
+
|
|
181
|
+
def policy(verdict: FailureVerdict, ctx: FailureContext) -> FailureAction:
|
|
182
|
+
message = error_message(ctx.error)
|
|
183
|
+
if verdict.kind == "transient":
|
|
184
|
+
ceiling = min(max_s, base_s * 2 ** (ctx.attempt - 1))
|
|
185
|
+
return Retry(delay_s=random() * ceiling)
|
|
186
|
+
if verdict.kind == "bad_output":
|
|
187
|
+
return RetryWithHint(hint=f"The previous attempt produced unusable output: {message}")
|
|
188
|
+
if verdict.kind in ("needs_human", "over_budget"):
|
|
189
|
+
return Escalate(reason=message)
|
|
190
|
+
return Fail(reason=message)
|
|
191
|
+
|
|
192
|
+
return policy
|
evenkeel/jev.py
ADDED
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
"""JevClassifier: asks TypeSafe's Jev what kind of failure an error is, with the rule classifier as fallback.
|
|
2
|
+
Ported from keel's src/jev-classifier.ts. Needs the `jev` extra (typesafe-sdk): pip install evenkeel[jev]."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import asyncio
|
|
7
|
+
import json
|
|
8
|
+
import logging
|
|
9
|
+
from dataclasses import replace
|
|
10
|
+
from typing import Any
|
|
11
|
+
|
|
12
|
+
from .failure import FailureContext, FailureVerdict, RuleClassifier, error_message
|
|
13
|
+
|
|
14
|
+
logger = logging.getLogger(__name__)
|
|
15
|
+
|
|
16
|
+
_PREVIEW_CHARS = 2000
|
|
17
|
+
|
|
18
|
+
# over_budget is never asked: the middleware raises it explicitly.
|
|
19
|
+
_INSTRUCTIONS = (
|
|
20
|
+
"A step of an automated job, often an AI agent, failed with `error`. Decide what kind of failure it is, "
|
|
21
|
+
"so the job engine can retry, retry with a correction, stop, or ask a person. Judge from `error`, `task` "
|
|
22
|
+
"and `payload`, and from `step` (the step that was running) and `output` (what the model or tool produced) "
|
|
23
|
+
"when present."
|
|
24
|
+
)
|
|
25
|
+
_CRITERIA = {
|
|
26
|
+
"transient": (
|
|
27
|
+
"A temporary problem outside the job: rate limits, timeouts, overloaded or unavailable services, "
|
|
28
|
+
"dropped connections. Retrying the same request later is likely to succeed."
|
|
29
|
+
),
|
|
30
|
+
"bad_input": (
|
|
31
|
+
"The job's own input is invalid or cannot be processed: missing or malformed fields, values that fail "
|
|
32
|
+
"validation, content too long for the model's context window. Retrying with the same input will fail again."
|
|
33
|
+
),
|
|
34
|
+
"bad_output": (
|
|
35
|
+
"A model or tool produced unusable output: invalid JSON, a call to a tool or function that does not exist, "
|
|
36
|
+
"wrong or missing arguments, a response that breaks the required schema. Retrying with a correction to "
|
|
37
|
+
"the model is likely to help."
|
|
38
|
+
),
|
|
39
|
+
"needs_human": (
|
|
40
|
+
"The job cannot continue until a person decides: an action above an approval limit, a model refusal or "
|
|
41
|
+
"policy block that needs review, information or permission that only a person can provide."
|
|
42
|
+
),
|
|
43
|
+
"fatal": (
|
|
44
|
+
"A bug or permanent problem in the code or configuration: type errors, undefined values, invalid "
|
|
45
|
+
"credentials, missing configuration or resources. Retrying will not help."
|
|
46
|
+
),
|
|
47
|
+
}
|
|
48
|
+
_ASKED = frozenset(_CRITERIA)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _question() -> Any:
|
|
52
|
+
from typesafe_sdk import Choice
|
|
53
|
+
|
|
54
|
+
return Choice(instructions=_INSTRUCTIONS, criteria=dict(_CRITERIA))
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _read_answer(response: Any) -> tuple[str | None, float]:
|
|
58
|
+
answers = getattr(response, "answers", None) or {}
|
|
59
|
+
answer = answers.get("failure_kind")
|
|
60
|
+
choice = getattr(answer, "choice", None)
|
|
61
|
+
confidence = getattr(answer, "confidence", None)
|
|
62
|
+
return (
|
|
63
|
+
choice if isinstance(choice, str) else None,
|
|
64
|
+
float(confidence) if isinstance(confidence, (int, float)) else 0.0,
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _json(value: Any) -> str:
|
|
69
|
+
try:
|
|
70
|
+
return json.dumps(value, default=str)
|
|
71
|
+
except (TypeError, ValueError):
|
|
72
|
+
return str(value)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _preview(text: str) -> str:
|
|
76
|
+
return f"{text[:_PREVIEW_CHARS]}... (truncated)" if len(text) > _PREVIEW_CHARS else text
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _describe_cause(cause: BaseException) -> str:
|
|
80
|
+
return f"{type(cause).__name__}: {error_message(cause)}"
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def describe_failure(ctx: FailureContext) -> dict[str, Any]:
|
|
84
|
+
"""The state sent to Jev: the error with its HTTP status, code and cause, plus the run context."""
|
|
85
|
+
error = ctx.error
|
|
86
|
+
described: dict[str, Any] = {"name": type(error).__name__, "message": error_message(error)}
|
|
87
|
+
for attr in ("status_code", "status", "code"):
|
|
88
|
+
value = getattr(error, attr, None)
|
|
89
|
+
if value is not None:
|
|
90
|
+
described["status" if attr != "code" else "code"] = value
|
|
91
|
+
cause = error.__cause__ or error.__context__
|
|
92
|
+
if cause is not None:
|
|
93
|
+
described["cause"] = _describe_cause(cause)
|
|
94
|
+
state: dict[str, Any] = {
|
|
95
|
+
"task": ctx.task,
|
|
96
|
+
"attempt": f"{ctx.attempt} of {ctx.max_attempts}",
|
|
97
|
+
"error": described,
|
|
98
|
+
"payload": _preview(_json(ctx.payload)),
|
|
99
|
+
}
|
|
100
|
+
if ctx.step is not None:
|
|
101
|
+
state["step"] = ctx.step
|
|
102
|
+
if ctx.output is not None:
|
|
103
|
+
state["output"] = _preview(_json(ctx.output))
|
|
104
|
+
return state
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
class JevClassifier:
|
|
108
|
+
"""Cascade: explicit signals (rule confidence 1) are classified by rules for free. Everything else is
|
|
109
|
+
asked of Jev, falling back to rules when Jev is unsure, unusable or unavailable."""
|
|
110
|
+
|
|
111
|
+
def __init__(
|
|
112
|
+
self,
|
|
113
|
+
client: Any = None,
|
|
114
|
+
*,
|
|
115
|
+
async_client: Any = None,
|
|
116
|
+
min_confidence: float = 0.5,
|
|
117
|
+
model: str | None = None,
|
|
118
|
+
) -> None:
|
|
119
|
+
if client is None and async_client is None:
|
|
120
|
+
raise ValueError(
|
|
121
|
+
"JevClassifier needs a client: JevClassifier(TypeSafeClient()) or async_client=..."
|
|
122
|
+
)
|
|
123
|
+
self._rules = RuleClassifier()
|
|
124
|
+
self._client = client
|
|
125
|
+
self._async_client = async_client
|
|
126
|
+
self._min_confidence = min_confidence
|
|
127
|
+
self._extra = {"model": model} if model else {}
|
|
128
|
+
|
|
129
|
+
def classify(self, ctx: FailureContext) -> FailureVerdict:
|
|
130
|
+
by_rules = self._rules.classify(ctx)
|
|
131
|
+
if by_rules.confidence >= 1:
|
|
132
|
+
return by_rules
|
|
133
|
+
if self._client is None:
|
|
134
|
+
try:
|
|
135
|
+
asyncio.get_running_loop()
|
|
136
|
+
except RuntimeError:
|
|
137
|
+
return asyncio.run(self.aclassify(ctx))
|
|
138
|
+
# Sync code inside a running event loop (a notebook, an async server) cannot await Jev here.
|
|
139
|
+
return replace(by_rules, source="rules (no sync jev client)")
|
|
140
|
+
try:
|
|
141
|
+
response = self._client.system_one(
|
|
142
|
+
state=describe_failure(ctx), questions={"failure_kind": _question()}, **self._extra
|
|
143
|
+
)
|
|
144
|
+
except Exception as error: # an outage must not turn every failure into a classifier error
|
|
145
|
+
logger.warning("Jev classification failed, using rules: %s", error)
|
|
146
|
+
return replace(by_rules, source="rules (jev unavailable)")
|
|
147
|
+
return self._verdict(response, by_rules)
|
|
148
|
+
|
|
149
|
+
async def aclassify(self, ctx: FailureContext) -> FailureVerdict:
|
|
150
|
+
by_rules = self._rules.classify(ctx)
|
|
151
|
+
if by_rules.confidence >= 1:
|
|
152
|
+
return by_rules
|
|
153
|
+
if self._async_client is None:
|
|
154
|
+
return await asyncio.to_thread(self.classify, ctx)
|
|
155
|
+
try:
|
|
156
|
+
response = await self._async_client.system_one(
|
|
157
|
+
state=describe_failure(ctx), questions={"failure_kind": _question()}, **self._extra
|
|
158
|
+
)
|
|
159
|
+
except Exception as error:
|
|
160
|
+
logger.warning("Jev classification failed, using rules: %s", error)
|
|
161
|
+
return replace(by_rules, source="rules (jev unavailable)")
|
|
162
|
+
return self._verdict(response, by_rules)
|
|
163
|
+
|
|
164
|
+
def _verdict(self, response: Any, by_rules: FailureVerdict) -> FailureVerdict:
|
|
165
|
+
choice, confidence = _read_answer(response)
|
|
166
|
+
if choice not in _ASKED:
|
|
167
|
+
return replace(by_rules, source="rules (jev answer unusable)")
|
|
168
|
+
if confidence < self._min_confidence:
|
|
169
|
+
return replace(by_rules, source="rules (jev unsure)")
|
|
170
|
+
return FailureVerdict(choice, confidence, "jev") # type: ignore[arg-type]
|
evenkeel/middleware.py
ADDED
|
@@ -0,0 +1,399 @@
|
|
|
1
|
+
"""EvenKeelMiddleware: classified retries with hints, dollar budgets and escalation for LangChain agents."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import operator
|
|
7
|
+
import time
|
|
8
|
+
from collections.abc import Awaitable, Callable
|
|
9
|
+
from dataclasses import replace
|
|
10
|
+
from typing import Annotated, Any
|
|
11
|
+
|
|
12
|
+
from langchain_core.messages import AIMessage, HumanMessage, ToolMessage
|
|
13
|
+
from typing_extensions import NotRequired
|
|
14
|
+
|
|
15
|
+
from ._compat import (
|
|
16
|
+
AgentMiddleware,
|
|
17
|
+
AgentState,
|
|
18
|
+
Command,
|
|
19
|
+
ExtendedModelResponse,
|
|
20
|
+
GraphBubbleUp,
|
|
21
|
+
ModelRequest,
|
|
22
|
+
ModelResponse,
|
|
23
|
+
ToolCallRequest,
|
|
24
|
+
hook_config,
|
|
25
|
+
interrupt,
|
|
26
|
+
)
|
|
27
|
+
from ._core import (
|
|
28
|
+
AttemptRecord,
|
|
29
|
+
Decision,
|
|
30
|
+
attach_attempts,
|
|
31
|
+
decide,
|
|
32
|
+
escalation_payload,
|
|
33
|
+
parse_budget_resume,
|
|
34
|
+
parse_resume,
|
|
35
|
+
)
|
|
36
|
+
from .budget import PriceTable
|
|
37
|
+
from .failure import (
|
|
38
|
+
Classifier,
|
|
39
|
+
Escalate,
|
|
40
|
+
FailureContext,
|
|
41
|
+
FailurePolicy,
|
|
42
|
+
FailureVerdict,
|
|
43
|
+
Retry,
|
|
44
|
+
RetryWithHint,
|
|
45
|
+
RuleClassifier,
|
|
46
|
+
default_policy,
|
|
47
|
+
error_message,
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
class EvenKeelState(AgentState):
|
|
52
|
+
evenkeel_spent_usd: NotRequired[float]
|
|
53
|
+
evenkeel_budget_usd: NotRequired[float]
|
|
54
|
+
evenkeel_attempts: NotRequired[Annotated[list[AttemptRecord], operator.add]]
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
class _RetryRequest:
|
|
58
|
+
"""Internal signal from the decision helpers: try the call again with this request."""
|
|
59
|
+
|
|
60
|
+
def __init__(self, request: Any, delay_s: float = 0.0, *, escalated: bool = False) -> None:
|
|
61
|
+
self.request = request
|
|
62
|
+
self.delay_s = delay_s
|
|
63
|
+
self.escalated = escalated
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _ask(payload: dict[str, Any], parse: Callable[[Any], Any]) -> Any:
|
|
67
|
+
"""Pause for a person and parse the answer. A malformed answer asks again with the error instead of
|
|
68
|
+
raising: LangGraph keeps a resume value once given, so raising would replay it on every later resume."""
|
|
69
|
+
while True:
|
|
70
|
+
answer = interrupt(payload)
|
|
71
|
+
try:
|
|
72
|
+
return parse(answer)
|
|
73
|
+
except ValueError as error:
|
|
74
|
+
payload = {**payload, "error": str(error)}
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _with_hint(request: ModelRequest, hint: str) -> ModelRequest:
|
|
78
|
+
return request.override(messages=[*request.messages, HumanMessage(f"[evenkeel] {hint}")])
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
class EvenKeelMiddleware(AgentMiddleware):
|
|
82
|
+
"""Classifies each failed model or tool call and acts on it: back off, retry with a hint, fail, or
|
|
83
|
+
escalate to a person. Optionally tracks spend in dollars and pauses the run when over budget."""
|
|
84
|
+
|
|
85
|
+
state_schema = EvenKeelState
|
|
86
|
+
|
|
87
|
+
def __init__(
|
|
88
|
+
self,
|
|
89
|
+
*,
|
|
90
|
+
classifier: Classifier | None = None,
|
|
91
|
+
policy: FailurePolicy | None = None,
|
|
92
|
+
max_attempts: int = 4,
|
|
93
|
+
budget_usd: float | None = None,
|
|
94
|
+
prices: PriceTable | None = None,
|
|
95
|
+
task: str = "agent",
|
|
96
|
+
sleep: Callable[[float], None] = time.sleep,
|
|
97
|
+
asleep: Callable[[float], Awaitable[None]] = asyncio.sleep,
|
|
98
|
+
) -> None:
|
|
99
|
+
super().__init__()
|
|
100
|
+
if max_attempts < 1:
|
|
101
|
+
raise ValueError("max_attempts must be at least 1")
|
|
102
|
+
if budget_usd is not None and prices is None:
|
|
103
|
+
raise ValueError("budget_usd needs prices=PriceTable({...}) to turn tokens into dollars")
|
|
104
|
+
if budget_usd is not None and budget_usd <= 0:
|
|
105
|
+
raise ValueError("budget_usd must be above 0")
|
|
106
|
+
self._classifier = classifier or RuleClassifier()
|
|
107
|
+
self._policy = policy or default_policy()
|
|
108
|
+
self._max_attempts = max_attempts
|
|
109
|
+
self._budget_usd = budget_usd
|
|
110
|
+
self._prices = prices
|
|
111
|
+
self._task = task
|
|
112
|
+
self._sleep = sleep
|
|
113
|
+
self._asleep = asleep
|
|
114
|
+
|
|
115
|
+
# Shared helpers
|
|
116
|
+
|
|
117
|
+
def _classify(self, ctx: FailureContext) -> FailureVerdict:
|
|
118
|
+
try:
|
|
119
|
+
return self._classifier.classify(ctx)
|
|
120
|
+
except GraphBubbleUp:
|
|
121
|
+
raise
|
|
122
|
+
except Exception: # a broken classifier must never hide the user's own error
|
|
123
|
+
return replace(RuleClassifier().classify(ctx), source="rules (classifier error)")
|
|
124
|
+
|
|
125
|
+
async def _aclassify(self, ctx: FailureContext) -> FailureVerdict:
|
|
126
|
+
try:
|
|
127
|
+
return await self._classifier.aclassify(ctx)
|
|
128
|
+
except GraphBubbleUp:
|
|
129
|
+
raise
|
|
130
|
+
except Exception:
|
|
131
|
+
return replace(RuleClassifier().classify(ctx), source="rules (classifier error)")
|
|
132
|
+
|
|
133
|
+
def _context(
|
|
134
|
+
self, error: BaseException, *, step: str, attempt: int, payload: Any = None
|
|
135
|
+
) -> FailureContext:
|
|
136
|
+
return FailureContext(
|
|
137
|
+
error=error,
|
|
138
|
+
task=self._task,
|
|
139
|
+
attempt=attempt,
|
|
140
|
+
max_attempts=self._max_attempts,
|
|
141
|
+
step=step,
|
|
142
|
+
payload=payload,
|
|
143
|
+
output=getattr(error, "output", None),
|
|
144
|
+
)
|
|
145
|
+
|
|
146
|
+
def _model_next(
|
|
147
|
+
self,
|
|
148
|
+
original: ModelRequest,
|
|
149
|
+
current: ModelRequest,
|
|
150
|
+
ctx: FailureContext,
|
|
151
|
+
verdict: FailureVerdict,
|
|
152
|
+
decision: Decision,
|
|
153
|
+
records: list[AttemptRecord],
|
|
154
|
+
escalated: bool,
|
|
155
|
+
) -> _RetryRequest:
|
|
156
|
+
"""Turns a decision into the next request, or raises the original error. A call that still needs a
|
|
157
|
+
person after one approved retry fails, so LangGraph's replay on resume cannot loop forever."""
|
|
158
|
+
action = decision.action
|
|
159
|
+
if isinstance(action, Retry):
|
|
160
|
+
return _RetryRequest(current, action.delay_s, escalated=escalated)
|
|
161
|
+
if isinstance(action, RetryWithHint):
|
|
162
|
+
return _RetryRequest(_with_hint(original, action.hint), escalated=escalated)
|
|
163
|
+
if isinstance(action, Escalate) and not escalated:
|
|
164
|
+
resume = _ask(
|
|
165
|
+
escalation_payload(ctx, verdict, action.reason),
|
|
166
|
+
lambda answer: parse_resume(answer, call_type="model"),
|
|
167
|
+
)
|
|
168
|
+
if resume.decision == "retry":
|
|
169
|
+
request = _with_hint(original, resume.hint) if resume.hint else original
|
|
170
|
+
return _RetryRequest(request, escalated=True)
|
|
171
|
+
attach_attempts(ctx.error, records)
|
|
172
|
+
raise ctx.error
|
|
173
|
+
|
|
174
|
+
@staticmethod
|
|
175
|
+
def _model_result(response: ModelResponse, records: list[AttemptRecord]) -> Any:
|
|
176
|
+
if not records:
|
|
177
|
+
return response
|
|
178
|
+
return ExtendedModelResponse(
|
|
179
|
+
model_response=response, command=Command(update={"evenkeel_attempts": records})
|
|
180
|
+
)
|
|
181
|
+
|
|
182
|
+
# Model calls
|
|
183
|
+
|
|
184
|
+
def wrap_model_call(self, request: ModelRequest, handler: Callable[[ModelRequest], ModelResponse]) -> Any:
|
|
185
|
+
records: list[AttemptRecord] = []
|
|
186
|
+
current = request
|
|
187
|
+
attempt = 0
|
|
188
|
+
escalated = False
|
|
189
|
+
while True:
|
|
190
|
+
attempt += 1
|
|
191
|
+
try:
|
|
192
|
+
response = handler(current)
|
|
193
|
+
except GraphBubbleUp:
|
|
194
|
+
raise
|
|
195
|
+
except Exception as error:
|
|
196
|
+
ctx = self._context(error, step="model", attempt=attempt)
|
|
197
|
+
verdict = self._classify(ctx)
|
|
198
|
+
decision = decide(ctx, verdict, self._policy)
|
|
199
|
+
records.append(decision.record)
|
|
200
|
+
nxt = self._model_next(request, current, ctx, verdict, decision, records, escalated)
|
|
201
|
+
escalated = nxt.escalated
|
|
202
|
+
if nxt.delay_s:
|
|
203
|
+
self._sleep(nxt.delay_s)
|
|
204
|
+
current = nxt.request
|
|
205
|
+
continue
|
|
206
|
+
return self._model_result(response, records)
|
|
207
|
+
|
|
208
|
+
async def awrap_model_call(
|
|
209
|
+
self, request: ModelRequest, handler: Callable[[ModelRequest], Awaitable[ModelResponse]]
|
|
210
|
+
) -> Any:
|
|
211
|
+
records: list[AttemptRecord] = []
|
|
212
|
+
current = request
|
|
213
|
+
attempt = 0
|
|
214
|
+
escalated = False
|
|
215
|
+
while True:
|
|
216
|
+
attempt += 1
|
|
217
|
+
try:
|
|
218
|
+
response = await handler(current)
|
|
219
|
+
except GraphBubbleUp:
|
|
220
|
+
raise
|
|
221
|
+
except Exception as error:
|
|
222
|
+
ctx = self._context(error, step="model", attempt=attempt)
|
|
223
|
+
verdict = await self._aclassify(ctx)
|
|
224
|
+
decision = decide(ctx, verdict, self._policy)
|
|
225
|
+
records.append(decision.record)
|
|
226
|
+
nxt = self._model_next(request, current, ctx, verdict, decision, records, escalated)
|
|
227
|
+
escalated = nxt.escalated
|
|
228
|
+
if nxt.delay_s:
|
|
229
|
+
await self._asleep(nxt.delay_s)
|
|
230
|
+
current = nxt.request
|
|
231
|
+
continue
|
|
232
|
+
return self._model_result(response, records)
|
|
233
|
+
|
|
234
|
+
# Tool calls
|
|
235
|
+
|
|
236
|
+
def _tool_next(
|
|
237
|
+
self,
|
|
238
|
+
request: ToolCallRequest,
|
|
239
|
+
ctx: FailureContext,
|
|
240
|
+
verdict: FailureVerdict,
|
|
241
|
+
decision: Decision,
|
|
242
|
+
records: list[AttemptRecord],
|
|
243
|
+
escalated: bool,
|
|
244
|
+
) -> _RetryRequest | Command:
|
|
245
|
+
"""Retry the tool, hand the error back to the model, or raise the original error."""
|
|
246
|
+
call = request.tool_call
|
|
247
|
+
action = decision.action
|
|
248
|
+
|
|
249
|
+
def back_to_model(content: str, status: str = "error") -> Command:
|
|
250
|
+
records[-1] = {**records[-1], "action": "returned_to_model"}
|
|
251
|
+
message = ToolMessage(content=content, tool_call_id=call["id"], name=call["name"], status=status)
|
|
252
|
+
return Command(update={"messages": [message], "evenkeel_attempts": records})
|
|
253
|
+
|
|
254
|
+
if isinstance(action, Retry):
|
|
255
|
+
return _RetryRequest(request, action.delay_s, escalated=escalated)
|
|
256
|
+
if isinstance(action, RetryWithHint):
|
|
257
|
+
return back_to_model(f"Error: {error_message(ctx.error)}. {action.hint}")
|
|
258
|
+
if verdict.kind == "bad_input":
|
|
259
|
+
return back_to_model(
|
|
260
|
+
f"Error: {error_message(ctx.error)}. Fix the arguments and call the tool again."
|
|
261
|
+
)
|
|
262
|
+
if isinstance(action, Escalate) and not escalated:
|
|
263
|
+
resume = _ask(
|
|
264
|
+
escalation_payload(ctx, verdict, action.reason),
|
|
265
|
+
lambda answer: parse_resume(answer, call_type="tool"),
|
|
266
|
+
)
|
|
267
|
+
if resume.decision == "retry" and resume.hint:
|
|
268
|
+
# A tool cannot take a hint, so the reviewer's words go to the model, which can call again.
|
|
269
|
+
return back_to_model(f"Error: {error_message(ctx.error)}. A reviewer said: {resume.hint}")
|
|
270
|
+
if resume.decision == "retry":
|
|
271
|
+
return _RetryRequest(request, escalated=True)
|
|
272
|
+
if resume.decision == "skip":
|
|
273
|
+
records[-1] = {**records[-1], "action": "skipped_by_reviewer"}
|
|
274
|
+
message = ToolMessage(content=resume.message, tool_call_id=call["id"], name=call["name"])
|
|
275
|
+
return Command(update={"messages": [message], "evenkeel_attempts": records})
|
|
276
|
+
attach_attempts(ctx.error, records)
|
|
277
|
+
raise ctx.error
|
|
278
|
+
|
|
279
|
+
@staticmethod
|
|
280
|
+
def _tool_result(result: Any, records: list[AttemptRecord]) -> Any:
|
|
281
|
+
if not records:
|
|
282
|
+
return result
|
|
283
|
+
if isinstance(result, ToolMessage):
|
|
284
|
+
return Command(update={"messages": [result], "evenkeel_attempts": records})
|
|
285
|
+
if isinstance(result, Command) and isinstance(result.update, dict):
|
|
286
|
+
existing = list(result.update.get("evenkeel_attempts", []))
|
|
287
|
+
return Command(
|
|
288
|
+
graph=result.graph,
|
|
289
|
+
update={**result.update, "evenkeel_attempts": existing + records},
|
|
290
|
+
resume=result.resume,
|
|
291
|
+
goto=result.goto,
|
|
292
|
+
)
|
|
293
|
+
return result
|
|
294
|
+
|
|
295
|
+
def wrap_tool_call(self, request: ToolCallRequest, handler: Callable[[ToolCallRequest], Any]) -> Any:
|
|
296
|
+
records: list[AttemptRecord] = []
|
|
297
|
+
attempt = 0
|
|
298
|
+
escalated = False
|
|
299
|
+
while True:
|
|
300
|
+
attempt += 1
|
|
301
|
+
try:
|
|
302
|
+
result = handler(request)
|
|
303
|
+
except GraphBubbleUp:
|
|
304
|
+
raise
|
|
305
|
+
except Exception as error:
|
|
306
|
+
ctx = self._context(
|
|
307
|
+
error,
|
|
308
|
+
step=f"tool:{request.tool_call['name']}",
|
|
309
|
+
attempt=attempt,
|
|
310
|
+
payload=request.tool_call.get("args"),
|
|
311
|
+
)
|
|
312
|
+
verdict = self._classify(ctx)
|
|
313
|
+
decision = decide(ctx, verdict, self._policy)
|
|
314
|
+
records.append(decision.record)
|
|
315
|
+
nxt = self._tool_next(request, ctx, verdict, decision, records, escalated)
|
|
316
|
+
if isinstance(nxt, Command):
|
|
317
|
+
return nxt
|
|
318
|
+
escalated = nxt.escalated
|
|
319
|
+
if nxt.delay_s:
|
|
320
|
+
self._sleep(nxt.delay_s)
|
|
321
|
+
continue
|
|
322
|
+
return self._tool_result(result, records)
|
|
323
|
+
|
|
324
|
+
async def awrap_tool_call(
|
|
325
|
+
self, request: ToolCallRequest, handler: Callable[[ToolCallRequest], Awaitable[Any]]
|
|
326
|
+
) -> Any:
|
|
327
|
+
records: list[AttemptRecord] = []
|
|
328
|
+
attempt = 0
|
|
329
|
+
escalated = False
|
|
330
|
+
while True:
|
|
331
|
+
attempt += 1
|
|
332
|
+
try:
|
|
333
|
+
result = await handler(request)
|
|
334
|
+
except GraphBubbleUp:
|
|
335
|
+
raise
|
|
336
|
+
except Exception as error:
|
|
337
|
+
ctx = self._context(
|
|
338
|
+
error,
|
|
339
|
+
step=f"tool:{request.tool_call['name']}",
|
|
340
|
+
attempt=attempt,
|
|
341
|
+
payload=request.tool_call.get("args"),
|
|
342
|
+
)
|
|
343
|
+
verdict = await self._aclassify(ctx)
|
|
344
|
+
decision = decide(ctx, verdict, self._policy)
|
|
345
|
+
records.append(decision.record)
|
|
346
|
+
nxt = self._tool_next(request, ctx, verdict, decision, records, escalated)
|
|
347
|
+
if isinstance(nxt, Command):
|
|
348
|
+
return nxt
|
|
349
|
+
escalated = nxt.escalated
|
|
350
|
+
if nxt.delay_s:
|
|
351
|
+
await self._asleep(nxt.delay_s)
|
|
352
|
+
continue
|
|
353
|
+
return self._tool_result(result, records)
|
|
354
|
+
|
|
355
|
+
# Budget
|
|
356
|
+
|
|
357
|
+
def before_agent(self, state: EvenKeelState, runtime: Any) -> dict[str, Any] | None:
|
|
358
|
+
"""Each run starts with zero spend and the configured budget; state persists per thread."""
|
|
359
|
+
if self._prices is None:
|
|
360
|
+
return None
|
|
361
|
+
reset: dict[str, Any] = {"evenkeel_spent_usd": 0.0}
|
|
362
|
+
if self._budget_usd is not None:
|
|
363
|
+
reset["evenkeel_budget_usd"] = self._budget_usd
|
|
364
|
+
return reset
|
|
365
|
+
|
|
366
|
+
def after_model(self, state: EvenKeelState, runtime: Any) -> dict[str, Any] | None:
|
|
367
|
+
if self._prices is None:
|
|
368
|
+
return None
|
|
369
|
+
# The model's reply is the newest AIMessage; structured output can put a ToolMessage after it.
|
|
370
|
+
message = next((m for m in reversed(state["messages"]) if isinstance(m, AIMessage)), None)
|
|
371
|
+
if message is None:
|
|
372
|
+
return None
|
|
373
|
+
spent = state.get("evenkeel_spent_usd", 0.0) + self._prices.cost_usd(message)
|
|
374
|
+
return {"evenkeel_spent_usd": spent}
|
|
375
|
+
|
|
376
|
+
@hook_config(can_jump_to=["end"])
|
|
377
|
+
def before_model(self, state: EvenKeelState, runtime: Any) -> dict[str, Any] | None:
|
|
378
|
+
budget = state.get("evenkeel_budget_usd", self._budget_usd)
|
|
379
|
+
if budget is None:
|
|
380
|
+
return None
|
|
381
|
+
spent = state.get("evenkeel_spent_usd", 0.0)
|
|
382
|
+
if spent < budget:
|
|
383
|
+
return None
|
|
384
|
+
new_budget = _ask(
|
|
385
|
+
{
|
|
386
|
+
"type": "evenkeel_budget",
|
|
387
|
+
"spent_usd": round(spent, 6),
|
|
388
|
+
"budget_usd": budget,
|
|
389
|
+
"accepted": [{"approve": True, "budget_usd": "<more than spent_usd>"}, {"approve": False}],
|
|
390
|
+
},
|
|
391
|
+
lambda answer: parse_budget_resume(answer, spent_usd=spent),
|
|
392
|
+
)
|
|
393
|
+
if new_budget is not None:
|
|
394
|
+
return {"evenkeel_budget_usd": new_budget}
|
|
395
|
+
note = AIMessage(
|
|
396
|
+
f"Stopped by evenkeel: spent ${spent:.4f} of a ${budget:.4f} budget "
|
|
397
|
+
"and the increase was declined."
|
|
398
|
+
)
|
|
399
|
+
return {"messages": [note], "jump_to": "end"}
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: evenkeel
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Failure classification, retries with hints, dollar budgets and escalation for LangChain agents. Ported from keel.
|
|
5
|
+
Project-URL: Repository, https://github.com/yodablocks/evenkeel
|
|
6
|
+
Project-URL: keel, https://github.com/yodablocks/keel
|
|
7
|
+
Author: yodablocks
|
|
8
|
+
License: MIT
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Keywords: agents,budgets,human-in-the-loop,jev,langchain,langgraph,middleware,retries
|
|
11
|
+
Classifier: Development Status :: 3 - Alpha
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
19
|
+
Requires-Python: >=3.10
|
|
20
|
+
Requires-Dist: langchain<2,>=1.3
|
|
21
|
+
Requires-Dist: langgraph<2,>=1
|
|
22
|
+
Requires-Dist: typing-extensions>=4.7
|
|
23
|
+
Provides-Extra: jev
|
|
24
|
+
Requires-Dist: typesafe-sdk<0.8,>=0.7; extra == 'jev'
|
|
25
|
+
Description-Content-Type: text/markdown
|
|
26
|
+
|
|
27
|
+
# evenkeel
|
|
28
|
+
|
|
29
|
+
Keeps a LangChain agent on an even keel when things fail: failures are classified and handled by kind,
|
|
30
|
+
bad output is retried with a hint, spend is tracked in dollars with a budget, and failures that need a person
|
|
31
|
+
pause the run for approval.
|
|
32
|
+
|
|
33
|
+
Ported from [keel](https://github.com/yodablocks/keel), a durable execution engine for AI agents.
|
|
34
|
+
An independent project, not affiliated with LangChain. Experimental: 0.x, expect changes.
|
|
35
|
+
|
|
36
|
+
## What it adds to LangChain's built-in middleware
|
|
37
|
+
|
|
38
|
+
| Need | Built in | EvenKeelMiddleware |
|
|
39
|
+
|---|---|---|
|
|
40
|
+
| Retries | `ModelRetryMiddleware`, `ToolRetryMiddleware` retry with the same input | Classifies the failure first: back off, retry **with a hint**, fail at once, or escalate |
|
|
41
|
+
| Limits | `ModelCallLimitMiddleware` counts calls | Tracks **dollars** from token usage and pauses at a budget |
|
|
42
|
+
| Approvals | `HumanInTheLoopMiddleware` asks before chosen tools | Escalates **when a failure needs a person**, from any model or tool call |
|
|
43
|
+
|
|
44
|
+
Use it alongside the built-ins: `ModelFallbackMiddleware` for switching models, `HumanInTheLoopMiddleware`
|
|
45
|
+
for approvals fixed in advance.
|
|
46
|
+
|
|
47
|
+
## Install
|
|
48
|
+
|
|
49
|
+
pip install evenkeel # rule-based classification
|
|
50
|
+
pip install "evenkeel[jev]" # adds TypeSafe's Jev classifier
|
|
51
|
+
|
|
52
|
+
## Use
|
|
53
|
+
|
|
54
|
+
```python
|
|
55
|
+
from langchain.agents import create_agent
|
|
56
|
+
from langgraph.checkpoint.memory import InMemorySaver
|
|
57
|
+
from evenkeel import EvenKeelMiddleware, ModelPrice, PriceTable
|
|
58
|
+
|
|
59
|
+
agent = create_agent(
|
|
60
|
+
"openai:gpt-4o-mini",
|
|
61
|
+
tools=[...],
|
|
62
|
+
middleware=[
|
|
63
|
+
EvenKeelMiddleware(
|
|
64
|
+
budget_usd=0.50,
|
|
65
|
+
prices=PriceTable({"gpt-4o-mini": ModelPrice(input_per_mtok=0.15, output_per_mtok=0.60)}),
|
|
66
|
+
)
|
|
67
|
+
],
|
|
68
|
+
checkpointer=InMemorySaver(), # needed for pauses
|
|
69
|
+
)
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
With Jev (needs `TYPESAFE_API_KEY`):
|
|
73
|
+
|
|
74
|
+
```python
|
|
75
|
+
from typesafe_sdk import TypeSafeClient
|
|
76
|
+
from evenkeel import EvenKeelMiddleware, JevClassifier
|
|
77
|
+
|
|
78
|
+
EvenKeelMiddleware(classifier=JevClassifier(TypeSafeClient()))
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
## How failures are handled
|
|
82
|
+
|
|
83
|
+
| Kind | Example | Action |
|
|
84
|
+
|---|---|---|
|
|
85
|
+
| `transient` | 429, 503, timeout, dropped connection | Back off and retry the same call |
|
|
86
|
+
| `bad_output` | Malformed output, a call that breaks the schema (recognised by Jev, or raised as `BadOutputError`) | Retry with a hint saying what was wrong |
|
|
87
|
+
| `bad_input` | Input a tool rejects (`BadInputError`, HTTP 400 or 422, a pydantic `ValidationError`) | Model call: fail. Tool call: return the error to the model so it fixes its arguments |
|
|
88
|
+
| `needs_human` | Refund over a limit, a policy block | Pause with `interrupt()` and wait for a decision |
|
|
89
|
+
| `fatal` | A bug, bad credentials | Re-raise the original error at once |
|
|
90
|
+
|
|
91
|
+
Retries and escalations stop at `max_attempts` (default 4). Each failure is recorded (error, kind, confidence,
|
|
92
|
+
which classifier decided, action): in the agent's state under `evenkeel_attempts` when the call ends in a
|
|
93
|
+
result, and on the raised error as `error.evenkeel_attempts` when it finally fails.
|
|
94
|
+
|
|
95
|
+
Some failures never reach the middleware, because LangChain turns them into messages for the model itself:
|
|
96
|
+
a call to a tool that does not exist, and tool arguments that fail the tool's schema.
|
|
97
|
+
|
|
98
|
+
## Pauses and how to resume
|
|
99
|
+
|
|
100
|
+
A failure that needs a person:
|
|
101
|
+
|
|
102
|
+
```python
|
|
103
|
+
from langgraph.types import Command
|
|
104
|
+
|
|
105
|
+
agent.invoke(Command(resume={"decision": "retry", "hint": "approved by finance"}), config)
|
|
106
|
+
agent.invoke(Command(resume={"decision": "fail"}), config)
|
|
107
|
+
agent.invoke(Command(resume={"decision": "skip", "message": "Refunded by hand."}), config) # tool calls only
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
A `retry` with a `hint` on a tool call sends the reviewer's words to the model, which can call the tool again; a
|
|
111
|
+
tool cannot take a hint itself. A call that still needs a person after one approved retry fails. An answer
|
|
112
|
+
that is not one of these values does not raise: the run pauses again, with an `error` field in the payload
|
|
113
|
+
saying what was wrong.
|
|
114
|
+
|
|
115
|
+
Over budget:
|
|
116
|
+
|
|
117
|
+
```python
|
|
118
|
+
agent.invoke(Command(resume={"approve": True, "budget_usd": 1.00}), config)
|
|
119
|
+
agent.invoke(Command(resume={"approve": False}), config) # ends the run
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
The budget applies to each run: every new `invoke` on a thread starts with zero spend and the configured
|
|
123
|
+
budget.
|
|
124
|
+
|
|
125
|
+
## Limitations
|
|
126
|
+
|
|
127
|
+
- **Budgets can overshoot by one call.** The check runs before a model call, but a call's cost is known only
|
|
128
|
+
after it. keel has the same limitation.
|
|
129
|
+
- **No built-in prices.** You supply them; a model without a price, or a response without token usage, is an
|
|
130
|
+
error rather than a silent $0. Dated or versioned names (`gpt-4o-mini-2024-07-18`, `claude-3-5-sonnet-20241022`, `gpt-4-0613`, `...-latest`)
|
|
131
|
+
use the price of their base name; any other variant (`gpt-4o-mini` against `gpt-4`, `gpt-4-turbo`) needs its own
|
|
132
|
+
entry.
|
|
133
|
+
- **On resume after an escalation, the failing call runs once more** before the decision applies, because
|
|
134
|
+
LangGraph re-runs the paused node. For a tool with side effects, make it idempotent or use `skip`.
|
|
135
|
+
- **Classification is only as good as the classifier.** Rules only recognise explicit signals (status codes,
|
|
136
|
+
network errors, evenkeel's error classes): on keel's 75 error messages from public SDK issues they pick the
|
|
137
|
+
right kind 29 times and the right action 39 times. Jev reads the message itself; see
|
|
138
|
+
[keel's results](https://github.com/yodablocks/keel#failure-classification-rules-vs-jev) for its accuracy on
|
|
139
|
+
real error messages, and `evals/` to measure it here.
|
|
140
|
+
|
|
141
|
+
## Develop
|
|
142
|
+
|
|
143
|
+
uv sync --all-extras
|
|
144
|
+
uv run pytest
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
evenkeel/__init__.py,sha256=hMJ8o1T6b5qp37nY8ODmOnR0dJB_wRO6M_AHahmsK10,758
|
|
2
|
+
evenkeel/_compat.py,sha256=EnKgLg2l0iWWoPDMoSNRAptbfE3PRlGVEFvvkwMrCiw,678
|
|
3
|
+
evenkeel/_core.py,sha256=TfgFkOx0BjkIkVx0hhwoFV-1P6AacKUxFyuMPs0QRhc,4378
|
|
4
|
+
evenkeel/budget.py,sha256=6xrU8qjqfJXyUhvNpkT53sRPL9Q7rI8FDe468cRfCe8,2773
|
|
5
|
+
evenkeel/failure.py,sha256=LoX8Kq75ML5HjYzk9iMJHT1otiUm_poJpK88fy19uIc,6629
|
|
6
|
+
evenkeel/jev.py,sha256=_EW5cWYL52eGQKjM20bYm2JfXpAPG0PX_1gL4F8KAcc,7077
|
|
7
|
+
evenkeel/middleware.py,sha256=9e-mzVP7eL3Tm9wt1H4F12hGFRYxO9wYdbblxsFaQPQ,15540
|
|
8
|
+
evenkeel-0.1.0.dist-info/METADATA,sha256=ET_D-EcyQ5sQFD3QPfebBnkJc4wnpfTR20czkGtUD1w,6531
|
|
9
|
+
evenkeel-0.1.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
|
|
10
|
+
evenkeel-0.1.0.dist-info/licenses/LICENSE,sha256=7_aXYiLkWK_v-hMC4ckA6mS3BWVaLQnpAjV3h5dRmcI,1067
|
|
11
|
+
evenkeel-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 yodablocks
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|