evenkeel 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
evenkeel/__init__.py ADDED
@@ -0,0 +1,34 @@
1
+ """Failure classification, retries with hints, dollar budgets and escalation for LangChain agents."""
2
+
3
+ from ._core import AttemptRecord
4
+ from .budget import ModelPrice, PriceTable
5
+ from .failure import (
6
+ BadInputError,
7
+ BadOutputError,
8
+ FailureContext,
9
+ FailureVerdict,
10
+ NeedsHumanError,
11
+ OverBudgetError,
12
+ RuleClassifier,
13
+ default_policy,
14
+ )
15
+ from .jev import JevClassifier
16
+ from .middleware import EvenKeelMiddleware
17
+
18
+ __version__ = "0.1.0"
19
+
20
+ __all__ = [
21
+ "AttemptRecord",
22
+ "BadInputError",
23
+ "BadOutputError",
24
+ "EvenKeelMiddleware",
25
+ "FailureContext",
26
+ "FailureVerdict",
27
+ "JevClassifier",
28
+ "ModelPrice",
29
+ "NeedsHumanError",
30
+ "OverBudgetError",
31
+ "PriceTable",
32
+ "RuleClassifier",
33
+ "default_policy",
34
+ ]
evenkeel/_compat.py ADDED
@@ -0,0 +1,21 @@
1
+ """The one place that imports LangChain and LangGraph internals. If an import path moves in a new release,
2
+ change it here only."""
3
+
4
+ from langchain.agents.middleware import AgentMiddleware, AgentState, ModelRequest, ModelResponse, hook_config
5
+ from langchain.agents.middleware.types import ExtendedModelResponse
6
+ from langgraph.errors import GraphBubbleUp
7
+ from langgraph.prebuilt.tool_node import ToolCallRequest
8
+ from langgraph.types import Command, interrupt
9
+
10
+ __all__ = [
11
+ "AgentMiddleware",
12
+ "AgentState",
13
+ "Command",
14
+ "ExtendedModelResponse",
15
+ "GraphBubbleUp",
16
+ "ModelRequest",
17
+ "ModelResponse",
18
+ "ToolCallRequest",
19
+ "hook_config",
20
+ "interrupt",
21
+ ]
evenkeel/_core.py ADDED
@@ -0,0 +1,126 @@
1
+ """Decision logic shared by every integration: apply the policy, enforce the attempt limit, build records,
2
+ and read reviewers' resume values. No LangChain imports, so other integrations can reuse it."""
3
+
4
+ from __future__ import annotations
5
+
6
+ from dataclasses import dataclass
7
+ from typing import Any, Literal
8
+
9
+ from typing_extensions import TypedDict
10
+
11
+ from .failure import (
12
+ Escalate,
13
+ Fail,
14
+ FailureAction,
15
+ FailureContext,
16
+ FailurePolicy,
17
+ FailureVerdict,
18
+ Retry,
19
+ RetryWithHint,
20
+ error_message,
21
+ )
22
+
23
+ _RECORD_ERROR_CHARS = 500
24
+
25
+
26
+ class AttemptRecord(TypedDict):
27
+ attempt: int
28
+ call: str
29
+ error_type: str
30
+ error: str
31
+ kind: str
32
+ confidence: float
33
+ source: str
34
+ action: str
35
+
36
+
37
+ @dataclass(frozen=True)
38
+ class Decision:
39
+ action: FailureAction
40
+ record: AttemptRecord
41
+
42
+
43
+ def decide(ctx: FailureContext, verdict: FailureVerdict, policy: FailurePolicy) -> Decision:
44
+ action = policy(verdict, ctx)
45
+ if isinstance(action, (Retry, RetryWithHint, Escalate)) and ctx.attempt >= ctx.max_attempts:
46
+ action = Fail(reason=f"attempt limit reached ({ctx.max_attempts}): {error_message(ctx.error)}")
47
+ record = AttemptRecord(
48
+ attempt=ctx.attempt,
49
+ call=ctx.step or "",
50
+ error_type=type(ctx.error).__name__,
51
+ error=error_message(ctx.error)[:_RECORD_ERROR_CHARS],
52
+ kind=verdict.kind,
53
+ confidence=round(verdict.confidence, 3),
54
+ source=verdict.source,
55
+ action=action.type,
56
+ )
57
+ return Decision(action=action, record=record)
58
+
59
+
60
+ def escalation_payload(ctx: FailureContext, verdict: FailureVerdict, reason: str) -> dict[str, Any]:
61
+ is_tool = (ctx.step or "").startswith("tool:")
62
+ return {
63
+ "type": "evenkeel_escalation",
64
+ "call": ctx.step or "",
65
+ "error_type": type(ctx.error).__name__,
66
+ "error": error_message(ctx.error)[:_RECORD_ERROR_CHARS],
67
+ "kind": verdict.kind,
68
+ "confidence": round(verdict.confidence, 3),
69
+ "reason": reason,
70
+ "accepted": ["retry", "fail", "skip"] if is_tool else ["retry", "fail"],
71
+ }
72
+
73
+
74
+ @dataclass(frozen=True)
75
+ class Resume:
76
+ decision: Literal["retry", "fail", "skip"]
77
+ hint: str | None
78
+ message: str | None
79
+
80
+
81
+ _RESUME_HELP = (
82
+ 'Resume with {"decision": "retry"} (optionally with "hint": "..."), {"decision": "fail"}, '
83
+ 'or, for tool calls only, {"decision": "skip", "message": "..."}'
84
+ )
85
+
86
+
87
+ def parse_resume(value: Any, *, call_type: Literal["model", "tool"]) -> Resume:
88
+ if not isinstance(value, dict) or value.get("decision") not in ("retry", "fail", "skip"):
89
+ raise ValueError(f"Unrecognised escalation decision {value!r}. {_RESUME_HELP}")
90
+ decision = value["decision"]
91
+ hint = value.get("hint")
92
+ message = value.get("message")
93
+ if hint is not None and not isinstance(hint, str):
94
+ raise ValueError(f"The decision's hint must be a string, got {hint!r}. {_RESUME_HELP}")
95
+ if decision == "skip":
96
+ if call_type == "model":
97
+ raise ValueError(f"A model call cannot be skipped: it has no stand-in result. {_RESUME_HELP}")
98
+ if not isinstance(message, str):
99
+ raise ValueError(f"A skip decision needs a message for the model. {_RESUME_HELP}")
100
+ return Resume("skip", None, message)
101
+ return Resume(decision, hint, None)
102
+
103
+
104
+ _BUDGET_HELP = 'Resume with {"approve": true, "budget_usd": <more than spent>} or {"approve": false}'
105
+
106
+
107
+ def parse_budget_resume(value: Any, *, spent_usd: float) -> float | None:
108
+ if not isinstance(value, dict) or not isinstance(value.get("approve"), bool):
109
+ raise ValueError(f"Unrecognised budget decision {value!r}. {_BUDGET_HELP}")
110
+ if not value["approve"]:
111
+ return None
112
+ budget = value.get("budget_usd")
113
+ if not isinstance(budget, (int, float)) or isinstance(budget, bool) or budget <= spent_usd:
114
+ raise ValueError(
115
+ f"An approval needs a budget_usd above the ${spent_usd:.4f} already spent, "
116
+ f"got {budget!r}. {_BUDGET_HELP}"
117
+ )
118
+ return float(budget)
119
+
120
+
121
+ def attach_attempts(error: BaseException, records: list[AttemptRecord]) -> None:
122
+ try:
123
+ error.evenkeel_attempts = list(records) # type: ignore[attr-defined]
124
+ except AttributeError: # some built-in exceptions refuse new attributes
125
+ if hasattr(error, "add_note"): # Python 3.11+
126
+ error.add_note(f"evenkeel attempts: {records}")
evenkeel/budget.py ADDED
@@ -0,0 +1,74 @@
1
+ """Turns a model response's token usage into dollars, from prices the user supplies."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import re
6
+ from collections.abc import Mapping
7
+ from dataclasses import dataclass
8
+
9
+ from langchain_core.messages import AIMessage
10
+
11
+
12
+ class UnknownModelPriceError(ValueError):
13
+ """The response's model is not in the price table."""
14
+
15
+
16
+ class MissingUsageError(ValueError):
17
+ """The response reported no token usage, so its cost is unknown."""
18
+
19
+
20
+ @dataclass(frozen=True)
21
+ class ModelPrice:
22
+ input_per_mtok: float
23
+ """Dollars per million input tokens."""
24
+ output_per_mtok: float
25
+ """Dollars per million output tokens."""
26
+
27
+
28
+ def model_name_of(message: AIMessage) -> str | None:
29
+ metadata = message.response_metadata or {}
30
+ name = metadata.get("model_name") or metadata.get("model")
31
+ return name if isinstance(name, str) else None
32
+
33
+
34
+ # What may follow a priced name: only a date or version number (-2024-07-18, -20241022, -0613, -001) or the
35
+ # -latest alias. Anything else (-5-sonnet, -turbo, -1106-preview) is a different model with its own price.
36
+ _VERSION_SUFFIX = re.compile(r"-(\d{4}-\d{2}-\d{2}|\d{3,8}|latest)")
37
+
38
+
39
+ class PriceTable:
40
+ """Prices per model name. A name with a version or date suffix uses its base price
41
+ (gpt-4o-mini-2024-07-18 -> gpt-4o-mini); a different model never borrows another's price
42
+ (gpt-4o-mini does not use gpt-4, and gpt-4-turbo does not use gpt-4)."""
43
+
44
+ def __init__(self, prices: Mapping[str, ModelPrice]) -> None:
45
+ self._prices = dict(prices)
46
+
47
+ def price_for(self, model_name: str) -> ModelPrice:
48
+ if model_name in self._prices:
49
+ return self._prices[model_name]
50
+ matches = [
51
+ key
52
+ for key in self._prices
53
+ if model_name.startswith(key) and _VERSION_SUFFIX.fullmatch(model_name[len(key) :])
54
+ ]
55
+ if not matches:
56
+ raise UnknownModelPriceError(
57
+ f"No price for model {model_name!r}. Add it to PriceTable; known: {sorted(self._prices)}"
58
+ )
59
+ return self._prices[max(matches, key=len)]
60
+
61
+ def cost_usd(self, message: AIMessage) -> float:
62
+ name = model_name_of(message)
63
+ if name is None:
64
+ raise UnknownModelPriceError(
65
+ "The response does not say which model produced it; response_metadata keys: "
66
+ f"{sorted((message.response_metadata or {}).keys())}"
67
+ )
68
+ price = self.price_for(name)
69
+ usage = message.usage_metadata
70
+ if not usage:
71
+ raise MissingUsageError(f"Model {name!r} reported no token usage, so this call's cost is unknown")
72
+ return (
73
+ usage["input_tokens"] * price.input_per_mtok + usage["output_tokens"] * price.output_per_mtok
74
+ ) / 1e6
evenkeel/failure.py ADDED
@@ -0,0 +1,192 @@
1
+ """Failure kinds, the rule classifier and the default policy. Ported from keel's src/failure.ts."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import errno as _errno
6
+ import random as _random
7
+ import socket as _socket
8
+ from collections.abc import Callable
9
+ from dataclasses import dataclass, field
10
+ from typing import Any, Literal, Protocol
11
+
12
+ FailureKind = Literal["transient", "bad_input", "bad_output", "needs_human", "over_budget", "fatal"]
13
+
14
+
15
+ @dataclass(frozen=True)
16
+ class FailureContext:
17
+ error: BaseException
18
+ task: str
19
+ attempt: int
20
+ """The attempt that just failed, starting at 1."""
21
+ max_attempts: int
22
+ step: str | None = None
23
+ """What was running: "model" or "tool:<name>"."""
24
+ payload: Any = None
25
+ output: Any = None
26
+
27
+
28
+ @dataclass(frozen=True)
29
+ class FailureVerdict:
30
+ kind: FailureKind
31
+ confidence: float
32
+ """0 to 1. Rule matches on explicit signals are near 1, fallbacks are low."""
33
+ source: str = "rules"
34
+ """Which classifier decided, e.g. "rules", "jev", "rules (jev unavailable)"."""
35
+
36
+
37
+ class Classifier(Protocol):
38
+ def classify(self, ctx: FailureContext) -> FailureVerdict: ...
39
+
40
+ async def aclassify(self, ctx: FailureContext) -> FailureVerdict: ...
41
+
42
+
43
+ class EvenKeelError(Exception):
44
+ """Base for explicit failure signals. `output` is what the model or tool produced."""
45
+
46
+ def __init__(self, message: str, *, output: Any = None) -> None:
47
+ super().__init__(message)
48
+ self.output = output
49
+
50
+
51
+ class BadInputError(EvenKeelError):
52
+ """The input can never succeed, whatever the retry."""
53
+
54
+
55
+ class BadOutputError(EvenKeelError):
56
+ """A model or tool produced unusable output (hallucinated tool, malformed JSON)."""
57
+
58
+
59
+ class NeedsHumanError(EvenKeelError):
60
+ """A person has to decide before the run can continue."""
61
+
62
+
63
+ class OverBudgetError(EvenKeelError):
64
+ """The run has spent its budget."""
65
+
66
+
67
+ _TRANSIENT_NAMES = frozenset(
68
+ {
69
+ "APITimeoutError",
70
+ "APIConnectionError",
71
+ "ConnectTimeout",
72
+ "ReadTimeout",
73
+ "WriteTimeout",
74
+ "PoolTimeout",
75
+ "TimeoutException",
76
+ "ConnectError",
77
+ "ReadError",
78
+ "RemoteProtocolError",
79
+ }
80
+ )
81
+ _VALIDATION_NAMES = frozenset({"ValidationError"})
82
+ # Network error codes, as strings on `code` (keel's list) or as numbers on `errno` (OSError, socket.gaierror).
83
+ _TRANSIENT_CODES = frozenset(
84
+ {"ETIMEDOUT", "ECONNRESET", "ECONNREFUSED", "EPIPE", "EAI_AGAIN", "UND_ERR_SOCKET"}
85
+ )
86
+ _TRANSIENT_ERRNOS = frozenset({_errno.ETIMEDOUT, _errno.ECONNRESET, _errno.ECONNREFUSED, _errno.EPIPE})
87
+
88
+
89
+ def error_message(error: BaseException) -> str:
90
+ return str(error) or type(error).__name__
91
+
92
+
93
+ def _http_status(error: BaseException) -> int | None:
94
+ for attr in ("status_code", "status"):
95
+ value = getattr(error, attr, None)
96
+ if isinstance(value, int) and not isinstance(value, bool):
97
+ return value
98
+ value = getattr(getattr(error, "response", None), "status_code", None)
99
+ return value if isinstance(value, int) and not isinstance(value, bool) else None
100
+
101
+
102
+ def _is_network_error(error: BaseException) -> bool:
103
+ if isinstance(error, (TimeoutError, ConnectionError)) or type(error).__name__ in _TRANSIENT_NAMES:
104
+ return True
105
+ code, number = getattr(error, "code", None), getattr(error, "errno", None)
106
+ if isinstance(error, _socket.gaierror):
107
+ # Only DNS errors use EAI_* numbers; on macOS EAI_AGAIN equals errno.ENOENT (a missing file).
108
+ return number == _socket.EAI_AGAIN
109
+ return (isinstance(code, str) and code in _TRANSIENT_CODES) or number in _TRANSIENT_ERRNOS
110
+
111
+
112
+ class RuleClassifier:
113
+ """Classifies failures from explicit signals only: error classes, HTTP status, network errors."""
114
+
115
+ def classify(self, ctx: FailureContext) -> FailureVerdict:
116
+ error = ctx.error
117
+ if isinstance(error, BadInputError):
118
+ return FailureVerdict("bad_input", 1.0)
119
+ if isinstance(error, BadOutputError):
120
+ return FailureVerdict("bad_output", 1.0)
121
+ if isinstance(error, NeedsHumanError):
122
+ return FailureVerdict("needs_human", 1.0)
123
+ if isinstance(error, OverBudgetError):
124
+ return FailureVerdict("over_budget", 1.0)
125
+
126
+ status = _http_status(error)
127
+ if status is not None:
128
+ if status in (408, 429) or status >= 500:
129
+ return FailureVerdict("transient", 0.9)
130
+ if status in (400, 422):
131
+ return FailureVerdict("bad_input", 0.8)
132
+ return FailureVerdict("fatal", 0.8)
133
+ if _is_network_error(error):
134
+ return FailureVerdict("transient", 0.9)
135
+ # SDKs often wrap the network error: look one level down the chain.
136
+ cause = error.__cause__ or error.__context__
137
+ if cause is not None and _is_network_error(cause):
138
+ return FailureVerdict("transient", 0.9)
139
+ if type(error).__name__ in _VALIDATION_NAMES:
140
+ return FailureVerdict("bad_input", 0.7)
141
+ return FailureVerdict("fatal", 0.5)
142
+
143
+ async def aclassify(self, ctx: FailureContext) -> FailureVerdict:
144
+ return self.classify(ctx)
145
+
146
+
147
+ @dataclass(frozen=True)
148
+ class Retry:
149
+ delay_s: float
150
+ type: Literal["retry"] = field(default="retry", init=False)
151
+
152
+
153
+ @dataclass(frozen=True)
154
+ class RetryWithHint:
155
+ hint: str
156
+ type: Literal["retry_modified"] = field(default="retry_modified", init=False)
157
+
158
+
159
+ @dataclass(frozen=True)
160
+ class Escalate:
161
+ reason: str
162
+ type: Literal["escalate"] = field(default="escalate", init=False)
163
+
164
+
165
+ @dataclass(frozen=True)
166
+ class Fail:
167
+ reason: str
168
+ type: Literal["fail"] = field(default="fail", init=False)
169
+
170
+
171
+ FailureAction = Retry | RetryWithHint | Escalate | Fail
172
+ FailurePolicy = Callable[[FailureVerdict, FailureContext], FailureAction]
173
+
174
+
175
+ def default_policy(
176
+ *, base_s: float = 1.0, max_s: float = 60.0, random: Callable[[], float] = _random.random
177
+ ) -> FailurePolicy:
178
+ """Transient: exponential backoff with full jitter. Bad output: retry with the error as a hint.
179
+ Needs-human and over-budget: escalate. Bad input and fatal: fail."""
180
+
181
+ def policy(verdict: FailureVerdict, ctx: FailureContext) -> FailureAction:
182
+ message = error_message(ctx.error)
183
+ if verdict.kind == "transient":
184
+ ceiling = min(max_s, base_s * 2 ** (ctx.attempt - 1))
185
+ return Retry(delay_s=random() * ceiling)
186
+ if verdict.kind == "bad_output":
187
+ return RetryWithHint(hint=f"The previous attempt produced unusable output: {message}")
188
+ if verdict.kind in ("needs_human", "over_budget"):
189
+ return Escalate(reason=message)
190
+ return Fail(reason=message)
191
+
192
+ return policy
evenkeel/jev.py ADDED
@@ -0,0 +1,170 @@
1
+ """JevClassifier: asks TypeSafe's Jev what kind of failure an error is, with the rule classifier as fallback.
2
+ Ported from keel's src/jev-classifier.ts. Needs the `jev` extra (typesafe-sdk): pip install evenkeel[jev]."""
3
+
4
+ from __future__ import annotations
5
+
6
+ import asyncio
7
+ import json
8
+ import logging
9
+ from dataclasses import replace
10
+ from typing import Any
11
+
12
+ from .failure import FailureContext, FailureVerdict, RuleClassifier, error_message
13
+
14
+ logger = logging.getLogger(__name__)
15
+
16
+ _PREVIEW_CHARS = 2000
17
+
18
+ # over_budget is never asked: the middleware raises it explicitly.
19
+ _INSTRUCTIONS = (
20
+ "A step of an automated job, often an AI agent, failed with `error`. Decide what kind of failure it is, "
21
+ "so the job engine can retry, retry with a correction, stop, or ask a person. Judge from `error`, `task` "
22
+ "and `payload`, and from `step` (the step that was running) and `output` (what the model or tool produced) "
23
+ "when present."
24
+ )
25
+ _CRITERIA = {
26
+ "transient": (
27
+ "A temporary problem outside the job: rate limits, timeouts, overloaded or unavailable services, "
28
+ "dropped connections. Retrying the same request later is likely to succeed."
29
+ ),
30
+ "bad_input": (
31
+ "The job's own input is invalid or cannot be processed: missing or malformed fields, values that fail "
32
+ "validation, content too long for the model's context window. Retrying with the same input will fail again."
33
+ ),
34
+ "bad_output": (
35
+ "A model or tool produced unusable output: invalid JSON, a call to a tool or function that does not exist, "
36
+ "wrong or missing arguments, a response that breaks the required schema. Retrying with a correction to "
37
+ "the model is likely to help."
38
+ ),
39
+ "needs_human": (
40
+ "The job cannot continue until a person decides: an action above an approval limit, a model refusal or "
41
+ "policy block that needs review, information or permission that only a person can provide."
42
+ ),
43
+ "fatal": (
44
+ "A bug or permanent problem in the code or configuration: type errors, undefined values, invalid "
45
+ "credentials, missing configuration or resources. Retrying will not help."
46
+ ),
47
+ }
48
+ _ASKED = frozenset(_CRITERIA)
49
+
50
+
51
+ def _question() -> Any:
52
+ from typesafe_sdk import Choice
53
+
54
+ return Choice(instructions=_INSTRUCTIONS, criteria=dict(_CRITERIA))
55
+
56
+
57
+ def _read_answer(response: Any) -> tuple[str | None, float]:
58
+ answers = getattr(response, "answers", None) or {}
59
+ answer = answers.get("failure_kind")
60
+ choice = getattr(answer, "choice", None)
61
+ confidence = getattr(answer, "confidence", None)
62
+ return (
63
+ choice if isinstance(choice, str) else None,
64
+ float(confidence) if isinstance(confidence, (int, float)) else 0.0,
65
+ )
66
+
67
+
68
+ def _json(value: Any) -> str:
69
+ try:
70
+ return json.dumps(value, default=str)
71
+ except (TypeError, ValueError):
72
+ return str(value)
73
+
74
+
75
+ def _preview(text: str) -> str:
76
+ return f"{text[:_PREVIEW_CHARS]}... (truncated)" if len(text) > _PREVIEW_CHARS else text
77
+
78
+
79
+ def _describe_cause(cause: BaseException) -> str:
80
+ return f"{type(cause).__name__}: {error_message(cause)}"
81
+
82
+
83
+ def describe_failure(ctx: FailureContext) -> dict[str, Any]:
84
+ """The state sent to Jev: the error with its HTTP status, code and cause, plus the run context."""
85
+ error = ctx.error
86
+ described: dict[str, Any] = {"name": type(error).__name__, "message": error_message(error)}
87
+ for attr in ("status_code", "status", "code"):
88
+ value = getattr(error, attr, None)
89
+ if value is not None:
90
+ described["status" if attr != "code" else "code"] = value
91
+ cause = error.__cause__ or error.__context__
92
+ if cause is not None:
93
+ described["cause"] = _describe_cause(cause)
94
+ state: dict[str, Any] = {
95
+ "task": ctx.task,
96
+ "attempt": f"{ctx.attempt} of {ctx.max_attempts}",
97
+ "error": described,
98
+ "payload": _preview(_json(ctx.payload)),
99
+ }
100
+ if ctx.step is not None:
101
+ state["step"] = ctx.step
102
+ if ctx.output is not None:
103
+ state["output"] = _preview(_json(ctx.output))
104
+ return state
105
+
106
+
107
+ class JevClassifier:
108
+ """Cascade: explicit signals (rule confidence 1) are classified by rules for free. Everything else is
109
+ asked of Jev, falling back to rules when Jev is unsure, unusable or unavailable."""
110
+
111
+ def __init__(
112
+ self,
113
+ client: Any = None,
114
+ *,
115
+ async_client: Any = None,
116
+ min_confidence: float = 0.5,
117
+ model: str | None = None,
118
+ ) -> None:
119
+ if client is None and async_client is None:
120
+ raise ValueError(
121
+ "JevClassifier needs a client: JevClassifier(TypeSafeClient()) or async_client=..."
122
+ )
123
+ self._rules = RuleClassifier()
124
+ self._client = client
125
+ self._async_client = async_client
126
+ self._min_confidence = min_confidence
127
+ self._extra = {"model": model} if model else {}
128
+
129
+ def classify(self, ctx: FailureContext) -> FailureVerdict:
130
+ by_rules = self._rules.classify(ctx)
131
+ if by_rules.confidence >= 1:
132
+ return by_rules
133
+ if self._client is None:
134
+ try:
135
+ asyncio.get_running_loop()
136
+ except RuntimeError:
137
+ return asyncio.run(self.aclassify(ctx))
138
+ # Sync code inside a running event loop (a notebook, an async server) cannot await Jev here.
139
+ return replace(by_rules, source="rules (no sync jev client)")
140
+ try:
141
+ response = self._client.system_one(
142
+ state=describe_failure(ctx), questions={"failure_kind": _question()}, **self._extra
143
+ )
144
+ except Exception as error: # an outage must not turn every failure into a classifier error
145
+ logger.warning("Jev classification failed, using rules: %s", error)
146
+ return replace(by_rules, source="rules (jev unavailable)")
147
+ return self._verdict(response, by_rules)
148
+
149
+ async def aclassify(self, ctx: FailureContext) -> FailureVerdict:
150
+ by_rules = self._rules.classify(ctx)
151
+ if by_rules.confidence >= 1:
152
+ return by_rules
153
+ if self._async_client is None:
154
+ return await asyncio.to_thread(self.classify, ctx)
155
+ try:
156
+ response = await self._async_client.system_one(
157
+ state=describe_failure(ctx), questions={"failure_kind": _question()}, **self._extra
158
+ )
159
+ except Exception as error:
160
+ logger.warning("Jev classification failed, using rules: %s", error)
161
+ return replace(by_rules, source="rules (jev unavailable)")
162
+ return self._verdict(response, by_rules)
163
+
164
+ def _verdict(self, response: Any, by_rules: FailureVerdict) -> FailureVerdict:
165
+ choice, confidence = _read_answer(response)
166
+ if choice not in _ASKED:
167
+ return replace(by_rules, source="rules (jev answer unusable)")
168
+ if confidence < self._min_confidence:
169
+ return replace(by_rules, source="rules (jev unsure)")
170
+ return FailureVerdict(choice, confidence, "jev") # type: ignore[arg-type]
evenkeel/middleware.py ADDED
@@ -0,0 +1,399 @@
1
+ """EvenKeelMiddleware: classified retries with hints, dollar budgets and escalation for LangChain agents."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import asyncio
6
+ import operator
7
+ import time
8
+ from collections.abc import Awaitable, Callable
9
+ from dataclasses import replace
10
+ from typing import Annotated, Any
11
+
12
+ from langchain_core.messages import AIMessage, HumanMessage, ToolMessage
13
+ from typing_extensions import NotRequired
14
+
15
+ from ._compat import (
16
+ AgentMiddleware,
17
+ AgentState,
18
+ Command,
19
+ ExtendedModelResponse,
20
+ GraphBubbleUp,
21
+ ModelRequest,
22
+ ModelResponse,
23
+ ToolCallRequest,
24
+ hook_config,
25
+ interrupt,
26
+ )
27
+ from ._core import (
28
+ AttemptRecord,
29
+ Decision,
30
+ attach_attempts,
31
+ decide,
32
+ escalation_payload,
33
+ parse_budget_resume,
34
+ parse_resume,
35
+ )
36
+ from .budget import PriceTable
37
+ from .failure import (
38
+ Classifier,
39
+ Escalate,
40
+ FailureContext,
41
+ FailurePolicy,
42
+ FailureVerdict,
43
+ Retry,
44
+ RetryWithHint,
45
+ RuleClassifier,
46
+ default_policy,
47
+ error_message,
48
+ )
49
+
50
+
51
+ class EvenKeelState(AgentState):
52
+ evenkeel_spent_usd: NotRequired[float]
53
+ evenkeel_budget_usd: NotRequired[float]
54
+ evenkeel_attempts: NotRequired[Annotated[list[AttemptRecord], operator.add]]
55
+
56
+
57
+ class _RetryRequest:
58
+ """Internal signal from the decision helpers: try the call again with this request."""
59
+
60
+ def __init__(self, request: Any, delay_s: float = 0.0, *, escalated: bool = False) -> None:
61
+ self.request = request
62
+ self.delay_s = delay_s
63
+ self.escalated = escalated
64
+
65
+
66
+ def _ask(payload: dict[str, Any], parse: Callable[[Any], Any]) -> Any:
67
+ """Pause for a person and parse the answer. A malformed answer asks again with the error instead of
68
+ raising: LangGraph keeps a resume value once given, so raising would replay it on every later resume."""
69
+ while True:
70
+ answer = interrupt(payload)
71
+ try:
72
+ return parse(answer)
73
+ except ValueError as error:
74
+ payload = {**payload, "error": str(error)}
75
+
76
+
77
+ def _with_hint(request: ModelRequest, hint: str) -> ModelRequest:
78
+ return request.override(messages=[*request.messages, HumanMessage(f"[evenkeel] {hint}")])
79
+
80
+
81
+ class EvenKeelMiddleware(AgentMiddleware):
82
+ """Classifies each failed model or tool call and acts on it: back off, retry with a hint, fail, or
83
+ escalate to a person. Optionally tracks spend in dollars and pauses the run when over budget."""
84
+
85
+ state_schema = EvenKeelState
86
+
87
+ def __init__(
88
+ self,
89
+ *,
90
+ classifier: Classifier | None = None,
91
+ policy: FailurePolicy | None = None,
92
+ max_attempts: int = 4,
93
+ budget_usd: float | None = None,
94
+ prices: PriceTable | None = None,
95
+ task: str = "agent",
96
+ sleep: Callable[[float], None] = time.sleep,
97
+ asleep: Callable[[float], Awaitable[None]] = asyncio.sleep,
98
+ ) -> None:
99
+ super().__init__()
100
+ if max_attempts < 1:
101
+ raise ValueError("max_attempts must be at least 1")
102
+ if budget_usd is not None and prices is None:
103
+ raise ValueError("budget_usd needs prices=PriceTable({...}) to turn tokens into dollars")
104
+ if budget_usd is not None and budget_usd <= 0:
105
+ raise ValueError("budget_usd must be above 0")
106
+ self._classifier = classifier or RuleClassifier()
107
+ self._policy = policy or default_policy()
108
+ self._max_attempts = max_attempts
109
+ self._budget_usd = budget_usd
110
+ self._prices = prices
111
+ self._task = task
112
+ self._sleep = sleep
113
+ self._asleep = asleep
114
+
115
+ # Shared helpers
116
+
117
+ def _classify(self, ctx: FailureContext) -> FailureVerdict:
118
+ try:
119
+ return self._classifier.classify(ctx)
120
+ except GraphBubbleUp:
121
+ raise
122
+ except Exception: # a broken classifier must never hide the user's own error
123
+ return replace(RuleClassifier().classify(ctx), source="rules (classifier error)")
124
+
125
+ async def _aclassify(self, ctx: FailureContext) -> FailureVerdict:
126
+ try:
127
+ return await self._classifier.aclassify(ctx)
128
+ except GraphBubbleUp:
129
+ raise
130
+ except Exception:
131
+ return replace(RuleClassifier().classify(ctx), source="rules (classifier error)")
132
+
133
+ def _context(
134
+ self, error: BaseException, *, step: str, attempt: int, payload: Any = None
135
+ ) -> FailureContext:
136
+ return FailureContext(
137
+ error=error,
138
+ task=self._task,
139
+ attempt=attempt,
140
+ max_attempts=self._max_attempts,
141
+ step=step,
142
+ payload=payload,
143
+ output=getattr(error, "output", None),
144
+ )
145
+
146
+ def _model_next(
147
+ self,
148
+ original: ModelRequest,
149
+ current: ModelRequest,
150
+ ctx: FailureContext,
151
+ verdict: FailureVerdict,
152
+ decision: Decision,
153
+ records: list[AttemptRecord],
154
+ escalated: bool,
155
+ ) -> _RetryRequest:
156
+ """Turns a decision into the next request, or raises the original error. A call that still needs a
157
+ person after one approved retry fails, so LangGraph's replay on resume cannot loop forever."""
158
+ action = decision.action
159
+ if isinstance(action, Retry):
160
+ return _RetryRequest(current, action.delay_s, escalated=escalated)
161
+ if isinstance(action, RetryWithHint):
162
+ return _RetryRequest(_with_hint(original, action.hint), escalated=escalated)
163
+ if isinstance(action, Escalate) and not escalated:
164
+ resume = _ask(
165
+ escalation_payload(ctx, verdict, action.reason),
166
+ lambda answer: parse_resume(answer, call_type="model"),
167
+ )
168
+ if resume.decision == "retry":
169
+ request = _with_hint(original, resume.hint) if resume.hint else original
170
+ return _RetryRequest(request, escalated=True)
171
+ attach_attempts(ctx.error, records)
172
+ raise ctx.error
173
+
174
+ @staticmethod
175
+ def _model_result(response: ModelResponse, records: list[AttemptRecord]) -> Any:
176
+ if not records:
177
+ return response
178
+ return ExtendedModelResponse(
179
+ model_response=response, command=Command(update={"evenkeel_attempts": records})
180
+ )
181
+
182
+ # Model calls
183
+
184
+ def wrap_model_call(self, request: ModelRequest, handler: Callable[[ModelRequest], ModelResponse]) -> Any:
185
+ records: list[AttemptRecord] = []
186
+ current = request
187
+ attempt = 0
188
+ escalated = False
189
+ while True:
190
+ attempt += 1
191
+ try:
192
+ response = handler(current)
193
+ except GraphBubbleUp:
194
+ raise
195
+ except Exception as error:
196
+ ctx = self._context(error, step="model", attempt=attempt)
197
+ verdict = self._classify(ctx)
198
+ decision = decide(ctx, verdict, self._policy)
199
+ records.append(decision.record)
200
+ nxt = self._model_next(request, current, ctx, verdict, decision, records, escalated)
201
+ escalated = nxt.escalated
202
+ if nxt.delay_s:
203
+ self._sleep(nxt.delay_s)
204
+ current = nxt.request
205
+ continue
206
+ return self._model_result(response, records)
207
+
208
+ async def awrap_model_call(
209
+ self, request: ModelRequest, handler: Callable[[ModelRequest], Awaitable[ModelResponse]]
210
+ ) -> Any:
211
+ records: list[AttemptRecord] = []
212
+ current = request
213
+ attempt = 0
214
+ escalated = False
215
+ while True:
216
+ attempt += 1
217
+ try:
218
+ response = await handler(current)
219
+ except GraphBubbleUp:
220
+ raise
221
+ except Exception as error:
222
+ ctx = self._context(error, step="model", attempt=attempt)
223
+ verdict = await self._aclassify(ctx)
224
+ decision = decide(ctx, verdict, self._policy)
225
+ records.append(decision.record)
226
+ nxt = self._model_next(request, current, ctx, verdict, decision, records, escalated)
227
+ escalated = nxt.escalated
228
+ if nxt.delay_s:
229
+ await self._asleep(nxt.delay_s)
230
+ current = nxt.request
231
+ continue
232
+ return self._model_result(response, records)
233
+
234
+ # Tool calls
235
+
236
+ def _tool_next(
237
+ self,
238
+ request: ToolCallRequest,
239
+ ctx: FailureContext,
240
+ verdict: FailureVerdict,
241
+ decision: Decision,
242
+ records: list[AttemptRecord],
243
+ escalated: bool,
244
+ ) -> _RetryRequest | Command:
245
+ """Retry the tool, hand the error back to the model, or raise the original error."""
246
+ call = request.tool_call
247
+ action = decision.action
248
+
249
+ def back_to_model(content: str, status: str = "error") -> Command:
250
+ records[-1] = {**records[-1], "action": "returned_to_model"}
251
+ message = ToolMessage(content=content, tool_call_id=call["id"], name=call["name"], status=status)
252
+ return Command(update={"messages": [message], "evenkeel_attempts": records})
253
+
254
+ if isinstance(action, Retry):
255
+ return _RetryRequest(request, action.delay_s, escalated=escalated)
256
+ if isinstance(action, RetryWithHint):
257
+ return back_to_model(f"Error: {error_message(ctx.error)}. {action.hint}")
258
+ if verdict.kind == "bad_input":
259
+ return back_to_model(
260
+ f"Error: {error_message(ctx.error)}. Fix the arguments and call the tool again."
261
+ )
262
+ if isinstance(action, Escalate) and not escalated:
263
+ resume = _ask(
264
+ escalation_payload(ctx, verdict, action.reason),
265
+ lambda answer: parse_resume(answer, call_type="tool"),
266
+ )
267
+ if resume.decision == "retry" and resume.hint:
268
+ # A tool cannot take a hint, so the reviewer's words go to the model, which can call again.
269
+ return back_to_model(f"Error: {error_message(ctx.error)}. A reviewer said: {resume.hint}")
270
+ if resume.decision == "retry":
271
+ return _RetryRequest(request, escalated=True)
272
+ if resume.decision == "skip":
273
+ records[-1] = {**records[-1], "action": "skipped_by_reviewer"}
274
+ message = ToolMessage(content=resume.message, tool_call_id=call["id"], name=call["name"])
275
+ return Command(update={"messages": [message], "evenkeel_attempts": records})
276
+ attach_attempts(ctx.error, records)
277
+ raise ctx.error
278
+
279
+ @staticmethod
280
+ def _tool_result(result: Any, records: list[AttemptRecord]) -> Any:
281
+ if not records:
282
+ return result
283
+ if isinstance(result, ToolMessage):
284
+ return Command(update={"messages": [result], "evenkeel_attempts": records})
285
+ if isinstance(result, Command) and isinstance(result.update, dict):
286
+ existing = list(result.update.get("evenkeel_attempts", []))
287
+ return Command(
288
+ graph=result.graph,
289
+ update={**result.update, "evenkeel_attempts": existing + records},
290
+ resume=result.resume,
291
+ goto=result.goto,
292
+ )
293
+ return result
294
+
295
+ def wrap_tool_call(self, request: ToolCallRequest, handler: Callable[[ToolCallRequest], Any]) -> Any:
296
+ records: list[AttemptRecord] = []
297
+ attempt = 0
298
+ escalated = False
299
+ while True:
300
+ attempt += 1
301
+ try:
302
+ result = handler(request)
303
+ except GraphBubbleUp:
304
+ raise
305
+ except Exception as error:
306
+ ctx = self._context(
307
+ error,
308
+ step=f"tool:{request.tool_call['name']}",
309
+ attempt=attempt,
310
+ payload=request.tool_call.get("args"),
311
+ )
312
+ verdict = self._classify(ctx)
313
+ decision = decide(ctx, verdict, self._policy)
314
+ records.append(decision.record)
315
+ nxt = self._tool_next(request, ctx, verdict, decision, records, escalated)
316
+ if isinstance(nxt, Command):
317
+ return nxt
318
+ escalated = nxt.escalated
319
+ if nxt.delay_s:
320
+ self._sleep(nxt.delay_s)
321
+ continue
322
+ return self._tool_result(result, records)
323
+
324
+ async def awrap_tool_call(
325
+ self, request: ToolCallRequest, handler: Callable[[ToolCallRequest], Awaitable[Any]]
326
+ ) -> Any:
327
+ records: list[AttemptRecord] = []
328
+ attempt = 0
329
+ escalated = False
330
+ while True:
331
+ attempt += 1
332
+ try:
333
+ result = await handler(request)
334
+ except GraphBubbleUp:
335
+ raise
336
+ except Exception as error:
337
+ ctx = self._context(
338
+ error,
339
+ step=f"tool:{request.tool_call['name']}",
340
+ attempt=attempt,
341
+ payload=request.tool_call.get("args"),
342
+ )
343
+ verdict = await self._aclassify(ctx)
344
+ decision = decide(ctx, verdict, self._policy)
345
+ records.append(decision.record)
346
+ nxt = self._tool_next(request, ctx, verdict, decision, records, escalated)
347
+ if isinstance(nxt, Command):
348
+ return nxt
349
+ escalated = nxt.escalated
350
+ if nxt.delay_s:
351
+ await self._asleep(nxt.delay_s)
352
+ continue
353
+ return self._tool_result(result, records)
354
+
355
+ # Budget
356
+
357
+ def before_agent(self, state: EvenKeelState, runtime: Any) -> dict[str, Any] | None:
358
+ """Each run starts with zero spend and the configured budget; state persists per thread."""
359
+ if self._prices is None:
360
+ return None
361
+ reset: dict[str, Any] = {"evenkeel_spent_usd": 0.0}
362
+ if self._budget_usd is not None:
363
+ reset["evenkeel_budget_usd"] = self._budget_usd
364
+ return reset
365
+
366
+ def after_model(self, state: EvenKeelState, runtime: Any) -> dict[str, Any] | None:
367
+ if self._prices is None:
368
+ return None
369
+ # The model's reply is the newest AIMessage; structured output can put a ToolMessage after it.
370
+ message = next((m for m in reversed(state["messages"]) if isinstance(m, AIMessage)), None)
371
+ if message is None:
372
+ return None
373
+ spent = state.get("evenkeel_spent_usd", 0.0) + self._prices.cost_usd(message)
374
+ return {"evenkeel_spent_usd": spent}
375
+
376
+ @hook_config(can_jump_to=["end"])
377
+ def before_model(self, state: EvenKeelState, runtime: Any) -> dict[str, Any] | None:
378
+ budget = state.get("evenkeel_budget_usd", self._budget_usd)
379
+ if budget is None:
380
+ return None
381
+ spent = state.get("evenkeel_spent_usd", 0.0)
382
+ if spent < budget:
383
+ return None
384
+ new_budget = _ask(
385
+ {
386
+ "type": "evenkeel_budget",
387
+ "spent_usd": round(spent, 6),
388
+ "budget_usd": budget,
389
+ "accepted": [{"approve": True, "budget_usd": "<more than spent_usd>"}, {"approve": False}],
390
+ },
391
+ lambda answer: parse_budget_resume(answer, spent_usd=spent),
392
+ )
393
+ if new_budget is not None:
394
+ return {"evenkeel_budget_usd": new_budget}
395
+ note = AIMessage(
396
+ f"Stopped by evenkeel: spent ${spent:.4f} of a ${budget:.4f} budget "
397
+ "and the increase was declined."
398
+ )
399
+ return {"messages": [note], "jump_to": "end"}
@@ -0,0 +1,144 @@
1
+ Metadata-Version: 2.5
2
+ Name: evenkeel
3
+ Version: 0.1.0
4
+ Summary: Failure classification, retries with hints, dollar budgets and escalation for LangChain agents. Ported from keel.
5
+ Project-URL: Repository, https://github.com/yodablocks/evenkeel
6
+ Project-URL: keel, https://github.com/yodablocks/keel
7
+ Author: yodablocks
8
+ License: MIT
9
+ License-File: LICENSE
10
+ Keywords: agents,budgets,human-in-the-loop,jev,langchain,langgraph,middleware,retries
11
+ Classifier: Development Status :: 3 - Alpha
12
+ Classifier: Intended Audience :: Developers
13
+ Classifier: License :: OSI Approved :: MIT License
14
+ Classifier: Programming Language :: Python :: 3
15
+ Classifier: Programming Language :: Python :: 3.10
16
+ Classifier: Programming Language :: Python :: 3.11
17
+ Classifier: Programming Language :: Python :: 3.12
18
+ Classifier: Programming Language :: Python :: 3.13
19
+ Requires-Python: >=3.10
20
+ Requires-Dist: langchain<2,>=1.3
21
+ Requires-Dist: langgraph<2,>=1
22
+ Requires-Dist: typing-extensions>=4.7
23
+ Provides-Extra: jev
24
+ Requires-Dist: typesafe-sdk<0.8,>=0.7; extra == 'jev'
25
+ Description-Content-Type: text/markdown
26
+
27
+ # evenkeel
28
+
29
+ Keeps a LangChain agent on an even keel when things fail: failures are classified and handled by kind,
30
+ bad output is retried with a hint, spend is tracked in dollars with a budget, and failures that need a person
31
+ pause the run for approval.
32
+
33
+ Ported from [keel](https://github.com/yodablocks/keel), a durable execution engine for AI agents.
34
+ An independent project, not affiliated with LangChain. Experimental: 0.x, expect changes.
35
+
36
+ ## What it adds to LangChain's built-in middleware
37
+
38
+ | Need | Built in | EvenKeelMiddleware |
39
+ |---|---|---|
40
+ | Retries | `ModelRetryMiddleware`, `ToolRetryMiddleware` retry with the same input | Classifies the failure first: back off, retry **with a hint**, fail at once, or escalate |
41
+ | Limits | `ModelCallLimitMiddleware` counts calls | Tracks **dollars** from token usage and pauses at a budget |
42
+ | Approvals | `HumanInTheLoopMiddleware` asks before chosen tools | Escalates **when a failure needs a person**, from any model or tool call |
43
+
44
+ Use it alongside the built-ins: `ModelFallbackMiddleware` for switching models, `HumanInTheLoopMiddleware`
45
+ for approvals fixed in advance.
46
+
47
+ ## Install
48
+
49
+ pip install evenkeel # rule-based classification
50
+ pip install "evenkeel[jev]" # adds TypeSafe's Jev classifier
51
+
52
+ ## Use
53
+
54
+ ```python
55
+ from langchain.agents import create_agent
56
+ from langgraph.checkpoint.memory import InMemorySaver
57
+ from evenkeel import EvenKeelMiddleware, ModelPrice, PriceTable
58
+
59
+ agent = create_agent(
60
+ "openai:gpt-4o-mini",
61
+ tools=[...],
62
+ middleware=[
63
+ EvenKeelMiddleware(
64
+ budget_usd=0.50,
65
+ prices=PriceTable({"gpt-4o-mini": ModelPrice(input_per_mtok=0.15, output_per_mtok=0.60)}),
66
+ )
67
+ ],
68
+ checkpointer=InMemorySaver(), # needed for pauses
69
+ )
70
+ ```
71
+
72
+ With Jev (needs `TYPESAFE_API_KEY`):
73
+
74
+ ```python
75
+ from typesafe_sdk import TypeSafeClient
76
+ from evenkeel import EvenKeelMiddleware, JevClassifier
77
+
78
+ EvenKeelMiddleware(classifier=JevClassifier(TypeSafeClient()))
79
+ ```
80
+
81
+ ## How failures are handled
82
+
83
+ | Kind | Example | Action |
84
+ |---|---|---|
85
+ | `transient` | 429, 503, timeout, dropped connection | Back off and retry the same call |
86
+ | `bad_output` | Malformed output, a call that breaks the schema (recognised by Jev, or raised as `BadOutputError`) | Retry with a hint saying what was wrong |
87
+ | `bad_input` | Input a tool rejects (`BadInputError`, HTTP 400 or 422, a pydantic `ValidationError`) | Model call: fail. Tool call: return the error to the model so it fixes its arguments |
88
+ | `needs_human` | Refund over a limit, a policy block | Pause with `interrupt()` and wait for a decision |
89
+ | `fatal` | A bug, bad credentials | Re-raise the original error at once |
90
+
91
+ Retries and escalations stop at `max_attempts` (default 4). Each failure is recorded (error, kind, confidence,
92
+ which classifier decided, action): in the agent's state under `evenkeel_attempts` when the call ends in a
93
+ result, and on the raised error as `error.evenkeel_attempts` when it finally fails.
94
+
95
+ Some failures never reach the middleware, because LangChain turns them into messages for the model itself:
96
+ a call to a tool that does not exist, and tool arguments that fail the tool's schema.
97
+
98
+ ## Pauses and how to resume
99
+
100
+ A failure that needs a person:
101
+
102
+ ```python
103
+ from langgraph.types import Command
104
+
105
+ agent.invoke(Command(resume={"decision": "retry", "hint": "approved by finance"}), config)
106
+ agent.invoke(Command(resume={"decision": "fail"}), config)
107
+ agent.invoke(Command(resume={"decision": "skip", "message": "Refunded by hand."}), config) # tool calls only
108
+ ```
109
+
110
+ A `retry` with a `hint` on a tool call sends the reviewer's words to the model, which can call the tool again; a
111
+ tool cannot take a hint itself. A call that still needs a person after one approved retry fails. An answer
112
+ that is not one of these values does not raise: the run pauses again, with an `error` field in the payload
113
+ saying what was wrong.
114
+
115
+ Over budget:
116
+
117
+ ```python
118
+ agent.invoke(Command(resume={"approve": True, "budget_usd": 1.00}), config)
119
+ agent.invoke(Command(resume={"approve": False}), config) # ends the run
120
+ ```
121
+
122
+ The budget applies to each run: every new `invoke` on a thread starts with zero spend and the configured
123
+ budget.
124
+
125
+ ## Limitations
126
+
127
+ - **Budgets can overshoot by one call.** The check runs before a model call, but a call's cost is known only
128
+ after it. keel has the same limitation.
129
+ - **No built-in prices.** You supply them; a model without a price, or a response without token usage, is an
130
+ error rather than a silent $0. Dated or versioned names (`gpt-4o-mini-2024-07-18`, `claude-3-5-sonnet-20241022`, `gpt-4-0613`, `...-latest`)
131
+ use the price of their base name; any other variant (`gpt-4o-mini` against `gpt-4`, `gpt-4-turbo`) needs its own
132
+ entry.
133
+ - **On resume after an escalation, the failing call runs once more** before the decision applies, because
134
+ LangGraph re-runs the paused node. For a tool with side effects, make it idempotent or use `skip`.
135
+ - **Classification is only as good as the classifier.** Rules only recognise explicit signals (status codes,
136
+ network errors, evenkeel's error classes): on keel's 75 error messages from public SDK issues they pick the
137
+ right kind 29 times and the right action 39 times. Jev reads the message itself; see
138
+ [keel's results](https://github.com/yodablocks/keel#failure-classification-rules-vs-jev) for its accuracy on
139
+ real error messages, and `evals/` to measure it here.
140
+
141
+ ## Develop
142
+
143
+ uv sync --all-extras
144
+ uv run pytest
@@ -0,0 +1,11 @@
1
+ evenkeel/__init__.py,sha256=hMJ8o1T6b5qp37nY8ODmOnR0dJB_wRO6M_AHahmsK10,758
2
+ evenkeel/_compat.py,sha256=EnKgLg2l0iWWoPDMoSNRAptbfE3PRlGVEFvvkwMrCiw,678
3
+ evenkeel/_core.py,sha256=TfgFkOx0BjkIkVx0hhwoFV-1P6AacKUxFyuMPs0QRhc,4378
4
+ evenkeel/budget.py,sha256=6xrU8qjqfJXyUhvNpkT53sRPL9Q7rI8FDe468cRfCe8,2773
5
+ evenkeel/failure.py,sha256=LoX8Kq75ML5HjYzk9iMJHT1otiUm_poJpK88fy19uIc,6629
6
+ evenkeel/jev.py,sha256=_EW5cWYL52eGQKjM20bYm2JfXpAPG0PX_1gL4F8KAcc,7077
7
+ evenkeel/middleware.py,sha256=9e-mzVP7eL3Tm9wt1H4F12hGFRYxO9wYdbblxsFaQPQ,15540
8
+ evenkeel-0.1.0.dist-info/METADATA,sha256=ET_D-EcyQ5sQFD3QPfebBnkJc4wnpfTR20czkGtUD1w,6531
9
+ evenkeel-0.1.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
10
+ evenkeel-0.1.0.dist-info/licenses/LICENSE,sha256=7_aXYiLkWK_v-hMC4ckA6mS3BWVaLQnpAjV3h5dRmcI,1067
11
+ evenkeel-0.1.0.dist-info/RECORD,,
@@ -0,0 +1,4 @@
1
+ Wheel-Version: 1.0
2
+ Generator: hatchling 1.32.4
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 yodablocks
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.