durallm 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- durallm-0.2.0/LICENSE +21 -0
- durallm-0.2.0/PKG-INFO +368 -0
- durallm-0.2.0/README.md +336 -0
- durallm-0.2.0/pyproject.toml +89 -0
- durallm-0.2.0/setup.cfg +4 -0
- durallm-0.2.0/src/durallm/__init__.py +265 -0
- durallm-0.2.0/src/durallm/_env.py +13 -0
- durallm-0.2.0/src/durallm/agent/__init__.py +27 -0
- durallm-0.2.0/src/durallm/agent/context.py +274 -0
- durallm-0.2.0/src/durallm/agent/failover_plan.py +34 -0
- durallm-0.2.0/src/durallm/agent/idempotency.py +220 -0
- durallm-0.2.0/src/durallm/agent/state.py +124 -0
- durallm-0.2.0/src/durallm/agent/tool_validation.py +272 -0
- durallm-0.2.0/src/durallm/breaker/__init__.py +31 -0
- durallm-0.2.0/src/durallm/breaker/circuit_breaker.py +302 -0
- durallm-0.2.0/src/durallm/breaker/metrics.py +119 -0
- durallm-0.2.0/src/durallm/breaker/registry.py +53 -0
- durallm-0.2.0/src/durallm/breaker/state.py +28 -0
- durallm-0.2.0/src/durallm/capability/__init__.py +14 -0
- durallm-0.2.0/src/durallm/capability/profile.py +146 -0
- durallm-0.2.0/src/durallm/capability/registry.py +121 -0
- durallm-0.2.0/src/durallm/classifier.py +421 -0
- durallm-0.2.0/src/durallm/config.py +92 -0
- durallm-0.2.0/src/durallm/continuation/__init__.py +22 -0
- durallm-0.2.0/src/durallm/continuation/models.py +110 -0
- durallm-0.2.0/src/durallm/continuation/sqlite.py +268 -0
- durallm-0.2.0/src/durallm/continuation/store.py +236 -0
- durallm-0.2.0/src/durallm/demo.py +198 -0
- durallm-0.2.0/src/durallm/discovery.py +254 -0
- durallm-0.2.0/src/durallm/errors.py +155 -0
- durallm-0.2.0/src/durallm/execution/__init__.py +19 -0
- durallm-0.2.0/src/durallm/execution/deadline.py +76 -0
- durallm-0.2.0/src/durallm/execution/executor.py +885 -0
- durallm-0.2.0/src/durallm/execution/ledger.py +82 -0
- durallm-0.2.0/src/durallm/execution/policy.py +48 -0
- durallm-0.2.0/src/durallm/gateway.py +163 -0
- durallm-0.2.0/src/durallm/health/__init__.py +13 -0
- durallm-0.2.0/src/durallm/health/telemetry.py +205 -0
- durallm-0.2.0/src/durallm/mcp/__init__.py +13 -0
- durallm-0.2.0/src/durallm/mcp/proxy.py +303 -0
- durallm-0.2.0/src/durallm/models.py +120 -0
- durallm-0.2.0/src/durallm/observability/logger.py +91 -0
- durallm-0.2.0/src/durallm/pools.py +306 -0
- durallm-0.2.0/src/durallm/protocol/__init__.py +43 -0
- durallm-0.2.0/src/durallm/protocol/anthropic.py +268 -0
- durallm-0.2.0/src/durallm/protocol/gemini.py +268 -0
- durallm-0.2.0/src/durallm/protocol/ir.py +100 -0
- durallm-0.2.0/src/durallm/protocol/openai.py +307 -0
- durallm-0.2.0/src/durallm/providers/__init__.py +35 -0
- durallm-0.2.0/src/durallm/providers/adapters.py +547 -0
- durallm-0.2.0/src/durallm/providers/base.py +193 -0
- durallm-0.2.0/src/durallm/proxy.py +942 -0
- durallm-0.2.0/src/durallm/pruner.py +137 -0
- durallm-0.2.0/src/durallm/router.py +341 -0
- durallm-0.2.0/src/durallm/routing/__init__.py +28 -0
- durallm-0.2.0/src/durallm/routing/budget.py +71 -0
- durallm-0.2.0/src/durallm/routing/cache.py +113 -0
- durallm-0.2.0/src/durallm/routing/decision.py +79 -0
- durallm-0.2.0/src/durallm/routing/keys.py +138 -0
- durallm-0.2.0/src/durallm/routing/quality.py +103 -0
- durallm-0.2.0/src/durallm/routing/requirements.py +156 -0
- durallm-0.2.0/src/durallm/routing/resources.py +63 -0
- durallm-0.2.0/src/durallm/routing/router.py +332 -0
- durallm-0.2.0/src/durallm/routing/scorer.py +118 -0
- durallm-0.2.0/src/durallm/routing/tokenizer.py +73 -0
- durallm-0.2.0/src/durallm/security/defense.py +84 -0
- durallm-0.2.0/src/durallm/storage/__init__.py +16 -0
- durallm-0.2.0/src/durallm/storage/contracts.py +120 -0
- durallm-0.2.0/src/durallm/storage/sqlite.py +552 -0
- durallm-0.2.0/src/durallm/storage/tool_ledger.py +142 -0
- durallm-0.2.0/src/durallm/streaming/__init__.py +19 -0
- durallm-0.2.0/src/durallm/streaming/modes.py +190 -0
- durallm-0.2.0/src/durallm/streaming/parser.py +190 -0
- durallm-0.2.0/src/durallm/translators.py +341 -0
- durallm-0.2.0/src/durallm/validation/response.py +105 -0
- durallm-0.2.0/src/durallm.egg-info/PKG-INFO +368 -0
- durallm-0.2.0/src/durallm.egg-info/SOURCES.txt +105 -0
- durallm-0.2.0/src/durallm.egg-info/dependency_links.txt +1 -0
- durallm-0.2.0/src/durallm.egg-info/entry_points.txt +5 -0
- durallm-0.2.0/src/durallm.egg-info/requires.txt +18 -0
- durallm-0.2.0/src/durallm.egg-info/top_level.txt +2 -0
- durallm-0.2.0/src/llm_circuit_breaker/__init__.py +39 -0
- durallm-0.2.0/tests/test_benchmark_harness.py +214 -0
- durallm-0.2.0/tests/test_benchmark_run.py +98 -0
- durallm-0.2.0/tests/test_benchmark_scenarios.py +53 -0
- durallm-0.2.0/tests/test_capability_resolution.py +60 -0
- durallm-0.2.0/tests/test_classifier.py +36 -0
- durallm-0.2.0/tests/test_classifier_table.py +57 -0
- durallm-0.2.0/tests/test_continuation_protocol.py +140 -0
- durallm-0.2.0/tests/test_discovery.py +42 -0
- durallm-0.2.0/tests/test_durable_state.py +165 -0
- durallm-0.2.0/tests/test_import_side_effects.py +76 -0
- durallm-0.2.0/tests/test_native_streaming.py +265 -0
- durallm-0.2.0/tests/test_output_cap.py +91 -0
- durallm-0.2.0/tests/test_pools.py +63 -0
- durallm-0.2.0/tests/test_proxy.py +60 -0
- durallm-0.2.0/tests/test_proxy_failover_telemetry.py +98 -0
- durallm-0.2.0/tests/test_proxy_gateway.py +262 -0
- durallm-0.2.0/tests/test_proxy_http.py +95 -0
- durallm-0.2.0/tests/test_pruner.py +65 -0
- durallm-0.2.0/tests/test_readme_snippet.py +59 -0
- durallm-0.2.0/tests/test_requirements_enforcement.py +88 -0
- durallm-0.2.0/tests/test_router.py +53 -0
- durallm-0.2.0/tests/test_router_admission.py +77 -0
- durallm-0.2.0/tests/test_semantic_failover_benchmark.py +39 -0
- durallm-0.2.0/tests/test_translators.py +145 -0
- durallm-0.2.0/tests/test_vcr_wire_conformance.py +125 -0
durallm-0.2.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Deepak
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
durallm-0.2.0/PKG-INFO
ADDED
|
@@ -0,0 +1,368 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: durallm
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Durable, self-healing multi-provider LLM gateway and circuit breaker with autonomous failover, tool idempotency, and context compaction for AI agents (Claude Code, Hermes, OpenClaw, OpenCode).
|
|
5
|
+
Author-email: Deepak <deepak@users.noreply.github.com>
|
|
6
|
+
Project-URL: Homepage, https://github.com/d2epak/durallm
|
|
7
|
+
Project-URL: Bug Tracker, https://github.com/d2epak/durallm/issues
|
|
8
|
+
Classifier: Programming Language :: Python :: 3
|
|
9
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
10
|
+
Classifier: Operating System :: OS Independent
|
|
11
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
12
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
13
|
+
Requires-Python: >=3.9
|
|
14
|
+
Description-Content-Type: text/markdown
|
|
15
|
+
License-File: LICENSE
|
|
16
|
+
Provides-Extra: asgi
|
|
17
|
+
Requires-Dist: fastapi>=0.100.0; extra == "asgi"
|
|
18
|
+
Requires-Dist: uvicorn>=0.22.0; extra == "asgi"
|
|
19
|
+
Requires-Dist: httpx>=0.24.0; extra == "asgi"
|
|
20
|
+
Provides-Extra: proxy
|
|
21
|
+
Requires-Dist: fastapi>=0.100.0; extra == "proxy"
|
|
22
|
+
Requires-Dist: uvicorn>=0.22.0; extra == "proxy"
|
|
23
|
+
Requires-Dist: httpx>=0.24.0; extra == "proxy"
|
|
24
|
+
Provides-Extra: dev
|
|
25
|
+
Requires-Dist: pytest>=7.0.0; extra == "dev"
|
|
26
|
+
Requires-Dist: pytest-cov>=5.0.0; extra == "dev"
|
|
27
|
+
Requires-Dist: ruff>=0.8.0; extra == "dev"
|
|
28
|
+
Requires-Dist: mypy>=1.13.0; extra == "dev"
|
|
29
|
+
Requires-Dist: build>=1.2.2; extra == "dev"
|
|
30
|
+
Requires-Dist: litellm<1.91.0,>=1.90.0; extra == "dev"
|
|
31
|
+
Dynamic: license-file
|
|
32
|
+
|
|
33
|
+
<div align="center">
|
|
34
|
+
|
|
35
|
+
# โก LLM Circuit Breaker
|
|
36
|
+
|
|
37
|
+
**The Agent-Resilient Gateway for Autonomous AI Systems**
|
|
38
|
+
|
|
39
|
+
*Zero-loss semantic failover โข Idempotent tool ledger โข Formal 6-state FSM โข Protocol IR โข Diagnostic context compaction*
|
|
40
|
+
|
|
41
|
+
[](https://github.com/d2epak/llm-circuit-breaker/actions/workflows/ci.yml)
|
|
42
|
+
[](https://www.python.org/downloads/)
|
|
43
|
+
[](https://opensource.org/licenses/MIT)
|
|
44
|
+
[]()
|
|
45
|
+
[]()
|
|
46
|
+
[]()
|
|
47
|
+
[-100%25-blueviolet.svg)](docs/BENCHMARKS.md)
|
|
48
|
+
|
|
49
|
+
<br/>
|
|
50
|
+
|
|
51
|
+
[โก Instant Demo](#-instant-demo-zero-api-keys) โข [๐ Quickstart](#-quickstart) โข [๐ค Agent Drop-In](#-agent-drop-in-integration) โข [๐ง Core Architecture](#-architecture-the-6-core-pillars) โข [๐ Benchmarks](#-empirical-benchmarks-b1b15) โข [๐ฅ Comparison](#-architectural-comparison) โข [๐ Docs](#-documentation-hub)
|
|
52
|
+
|
|
53
|
+
</div>
|
|
54
|
+
|
|
55
|
+
---
|
|
56
|
+
|
|
57
|
+
## ๐ฅ Why Standard Proxies Break Autonomous Agents
|
|
58
|
+
|
|
59
|
+
Modern LLM proxies (**LiteLLM**, **Portkey**, **Cloudflare AI Gateway**) were architected for stateless chat completions. When paired with **autonomous agent loops** (**Claude Code**, **Hermes Agent**, **Cursor**, **Aider**, **OpenClaw**), standard proxies cause silent task corruption:
|
|
60
|
+
|
|
61
|
+
| Failure Mode | Standard Reverse Proxy Behavior | **LLM Circuit Breaker** Resolution |
|
|
62
|
+
|---|---|---|
|
|
63
|
+
| **Ghost Side-Effects** *(Replay Hazard)* | On upstream 5xx or disconnect, blindly resends payload. A destructive tool call (`execute_bash("rm -rf ...")` or database mutation) executes twice. | **Idempotent Tool Execution Ledger**: Stages calls through `PROPOSED` $\to$ `VALIDATED` $\to$ `SUBMITTED` $\to$ `COMMITTED`. Cached receipts suppress duplicate executions during retries. |
|
|
64
|
+
| **Context Window Overflow** | Failing over from a 128k context provider to a 32k provider triggers HTTP 400. Proxies blindly truncate from the head, erasing system prompts and root instructions. | **Diagnostic Context Compaction**: Preserves root user goal and system prompt; summarizes intermediate tool logs into structured diagnostics (exit codes, error snippets). |
|
|
65
|
+
| **Protocol Incompatibility** | Blindly forwards raw JSON payloads. Anthropic-formatted tools crash when sent to OpenAI or Gemini endpoints. | **Protocol Intermediate Representation (IR)**: Universal translation across Anthropic (`/v1/messages`), OpenAI (`/v1/chat/completions`), and Gemini schemas. |
|
|
66
|
+
| **Cascade Outages** | Simple cooldown timers or naive retry loops hammer failing endpoints, triggering exponential rate-limit penalties across clusters. | **Formal 6-State Circuit Breaker FSM**: Count- and time-based sliding windows with strictly bounded half-open probe permits (`half_open_active <= max_calls`). |
|
|
67
|
+
| **Mid-Stream Model Splicing** | Drops connection mid-stream and blindly switches providers, generating half-OpenAI / half-Anthropic token gibberish. | **Interruption Boundary Protection**: True streaming emits an explicit interruption boundary event rather than splicing tokens mid-flight. |
|
|
68
|
+
| **Cross-Pool Quota Exhaustion** | A rate limit in an exploratory coding agent poisons shared credentials for critical production workloads. | **Calibrated Task Selection**: Independent `ResourceLaneStore` isolates rate limits per credential, model, and pool with atomic pre-dispatch reservations. |
|
|
69
|
+
|
|
70
|
+
---
|
|
71
|
+
|
|
72
|
+
## โก Instant Demo (Zero API Keys)
|
|
73
|
+
|
|
74
|
+
Simulate provider outages, semantic failover, circuit tripping, and self-healing recovery in under **2 seconds** without installing dependencies or setting API keys:
|
|
75
|
+
|
|
76
|
+
```bash
|
|
77
|
+
python -m durallm.demo
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
```text
|
|
81
|
+
===========================================================================
|
|
82
|
+
โก LLM CIRCUIT BREAKER โ DETERMINISTIC RESILIENCE & SEMANTIC FAILOVER DEMO
|
|
83
|
+
===========================================================================
|
|
84
|
+
โถ STEP 1: Dispatching turn to Primary Provider (Cerebras)...
|
|
85
|
+
โ Result: Primary response: Tool code executed successfully
|
|
86
|
+
โ Selected Endpoint: primary-cerebras (Attempts: 1) | State: CLOSED
|
|
87
|
+
|
|
88
|
+
โถ STEP 2: Primary suffers 503 Outage; Gateway initiates Semantic Failover...
|
|
89
|
+
โ Failover Succeeded! Response: Secondary (Groq) fallback response
|
|
90
|
+
โ Primary Breaker State: OPEN (Tripped by 503 server errors)
|
|
91
|
+
โ Observable FailoverPlan: primary-cerebras -> secondary-groq (Reason: overloaded)
|
|
92
|
+
|
|
93
|
+
โถ STEP 3: Next Request arrives while Primary is OPEN...
|
|
94
|
+
โ Dispatched directly to: secondary-groq (Primary bypassed with 0 upstream load)
|
|
95
|
+
|
|
96
|
+
โถ STEP 4: Advancing clock by 20 seconds; Testing Self-Healing Recovery...
|
|
97
|
+
โ Evaluated Breaker State: HALF_OPEN (Admits bounded probe permits)
|
|
98
|
+
โ Probe calls succeed -> Breaker Reset! Primary State is now: CLOSED
|
|
99
|
+
===========================================================================
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
---
|
|
103
|
+
|
|
104
|
+
## ๐๏ธ System Architecture
|
|
105
|
+
|
|
106
|
+
```mermaid
|
|
107
|
+
flowchart TD
|
|
108
|
+
subgraph Agents ["Autonomous Agent Clients"]
|
|
109
|
+
CC[Claude Code]
|
|
110
|
+
HA[Hermes Agent]
|
|
111
|
+
OC[OpenClaw]
|
|
112
|
+
CU[Cursor IDE]
|
|
113
|
+
AI[Aider]
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
subgraph Edge ["Protocol Edge (Zero Core Dependencies)"]
|
|
117
|
+
P1["/v1/messages (Anthropic REST/SSE)"]
|
|
118
|
+
P2["/v1/chat/completions (OpenAI REST/SSE)"]
|
|
119
|
+
ACP["Agent Continuation Protocol (ACP v1)"]
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
subgraph Runtime ["LLM Circuit Breaker Gateway Runtime"]
|
|
123
|
+
CB["1. Circuit Breaker FSM\n(CLOSED / OPEN / HALF_OPEN / FORCED_OPEN)"]
|
|
124
|
+
IR["2. Protocol IR\n(Universal Schema Translator)"]
|
|
125
|
+
TL["3. Idempotent Tool Ledger\n(PROPOSED โ VALIDATED โ COMMITTED)"]
|
|
126
|
+
CCMP["4. Diagnostic Context Compactor\n(Preserve Root + Exit-Code Extraction)"]
|
|
127
|
+
ROUT["5. Calibrated Task Selection\n(Privacy Tiers + Resource Lanes)"]
|
|
128
|
+
WAL[("6. Durable State Store\n(SQLite WAL Persistence)")]
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
subgraph Upstreams ["Upstream Inference Providers"]
|
|
132
|
+
U1["Anthropic\n(Claude 3.5 Sonnet)"]
|
|
133
|
+
U2["OpenAI\n(GPT-4o / o3-mini)"]
|
|
134
|
+
U3["Groq / Cerebras\n(Ultra-Low Latency)"]
|
|
135
|
+
U4["Local vLLM / Ollama\n(Air-Gapped / Privacy Tiers)"]
|
|
136
|
+
U5["DeepSeek / OpenRouter\n(Cost-Optimized Fallbacks)"]
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
Agents --> Edge
|
|
140
|
+
Edge --> Runtime
|
|
141
|
+
CB <--> WAL
|
|
142
|
+
TL <--> WAL
|
|
143
|
+
ACP <--> WAL
|
|
144
|
+
Runtime --> Upstreams
|
|
145
|
+
```
|
|
146
|
+
|
|
147
|
+
---
|
|
148
|
+
|
|
149
|
+
## ๐ Quickstart
|
|
150
|
+
|
|
151
|
+
### 1. Installation
|
|
152
|
+
|
|
153
|
+
Install the package directly (requires **Python 3.10+**):
|
|
154
|
+
|
|
155
|
+
```bash
|
|
156
|
+
pip install llm-circuit-breaker
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
*(Zero third-party core dependencies. The base package runs purely on the Python standard library with optional SQLite WAL durability).*
|
|
160
|
+
|
|
161
|
+
### 2. Launch the Local Proxy Gateway
|
|
162
|
+
|
|
163
|
+
Start the resilience proxy locally on port 4001:
|
|
164
|
+
|
|
165
|
+
```bash
|
|
166
|
+
llm-proxy --port 4001
|
|
167
|
+
# Or run as a module:
|
|
168
|
+
python -m durallm.proxy --port 4001
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
By default, the proxy runs fully isolated. If you want automatic credential discovery from local environment files, use `--discover`:
|
|
172
|
+
|
|
173
|
+
```bash
|
|
174
|
+
llm-proxy --port 4001 --discover
|
|
175
|
+
```
|
|
176
|
+
|
|
177
|
+
---
|
|
178
|
+
|
|
179
|
+
## ๐ค Agent Drop-In Integration
|
|
180
|
+
|
|
181
|
+
Seamlessly point your favorite autonomous agent at `llm-circuit-breaker` by overriding the base URL:
|
|
182
|
+
|
|
183
|
+
### Claude Code
|
|
184
|
+
```bash
|
|
185
|
+
export ANTHROPIC_BASE_URL="http://127.0.0.1:4001"
|
|
186
|
+
claude
|
|
187
|
+
```
|
|
188
|
+
|
|
189
|
+
### Hermes Agent / OpenClaw
|
|
190
|
+
```bash
|
|
191
|
+
export OPENAI_BASE_URL="http://127.0.0.1:4001/v1"
|
|
192
|
+
export OPENAI_API_KEY="sk-dummy" # Gateway manages actual provider credentials
|
|
193
|
+
hermes
|
|
194
|
+
```
|
|
195
|
+
|
|
196
|
+
### Cursor IDE
|
|
197
|
+
Navigate to **Cursor Settings** $\to$ **Models** $\to$ **OpenAI API Key**:
|
|
198
|
+
- Check **Override OpenAI Base URL**
|
|
199
|
+
- Set Base URL: `http://127.0.0.1:4001/v1`
|
|
200
|
+
|
|
201
|
+
### Aider
|
|
202
|
+
```bash
|
|
203
|
+
aider --openai-api-base http://127.0.0.1:4001/v1 --model openai/gpt-4o
|
|
204
|
+
```
|
|
205
|
+
|
|
206
|
+
---
|
|
207
|
+
|
|
208
|
+
## ๐ Python SDK Usage
|
|
209
|
+
|
|
210
|
+
Use the deterministic gateway directly within Python agent applications:
|
|
211
|
+
|
|
212
|
+
```python
|
|
213
|
+
import os
|
|
214
|
+
|
|
215
|
+
from durallm import Endpoint, GatewayExecutor, ModelProfile, NormalizedMessage, NormalizedRequest
|
|
216
|
+
|
|
217
|
+
executor = GatewayExecutor()
|
|
218
|
+
|
|
219
|
+
# Nothing is registered by default: declare at least one endpoint in the pool you will call.
|
|
220
|
+
executor.capability_registry.register_endpoint(Endpoint(
|
|
221
|
+
id="groq-llama",
|
|
222
|
+
provider="groq",
|
|
223
|
+
model="llama-3.3-70b-versatile",
|
|
224
|
+
base_url="https://api.groq.com/openai/v1",
|
|
225
|
+
env_key="GROQ_API_KEY", # name of the key looked up in `api_keys` below
|
|
226
|
+
pool="coding",
|
|
227
|
+
profile=ModelProfile("groq", "llama-3.3-70b-versatile", context_window=131072, supports_tools=True),
|
|
228
|
+
))
|
|
229
|
+
|
|
230
|
+
request = NormalizedRequest(
|
|
231
|
+
model="default",
|
|
232
|
+
messages=[NormalizedMessage(role="user", content="Deploy application")],
|
|
233
|
+
)
|
|
234
|
+
|
|
235
|
+
response, decision, ledger = executor.execute(
|
|
236
|
+
request,
|
|
237
|
+
pool="coding",
|
|
238
|
+
strategy="reliability_aware",
|
|
239
|
+
api_keys={"GROQ_API_KEY": os.environ["GROQ_API_KEY"]},
|
|
240
|
+
)
|
|
241
|
+
print(f"Selected Endpoint: {decision.selected_endpoint.id}")
|
|
242
|
+
print(f"Response: {response.content}")
|
|
243
|
+
```
|
|
244
|
+
|
|
245
|
+
---
|
|
246
|
+
|
|
247
|
+
## ๐ง Architecture: The 6 Core Pillars
|
|
248
|
+
|
|
249
|
+
### 1. Formal 6-State Circuit Breaker FSM
|
|
250
|
+
Implements an industrial-grade finite state machine (`CLOSED`, `OPEN`, `HALF_OPEN`, `FORCED_OPEN`, `DISABLED`, `METRICS_ONLY`) with:
|
|
251
|
+
- **Time- and count-based sliding error windows**: Evaluates failure rate thresholds without bias from stale errors.
|
|
252
|
+
- **Permanent Error Taxonomy & Dead-List Pruning**: Distinguishes permanent configuration & lifecycle errors (401 Bad Key, 402 Out of Credits, 404/410 EOL) from transient network failures (429, 503). Instantly blacklists dead models pre-flight to short-circuit future failing HTTP calls.
|
|
253
|
+
- **Failover Telemetry & Transparency**: Surfaces `X-LCB-Failover`, `X-LCB-Active-Model`, `X-LCB-Selected-Endpoint` HTTP headers and attaches `lcb_failover` payload metadata so agents/UIs know when failovers occur.
|
|
254
|
+
- **Bounded Half-Open Probes**: Strictly enforces `active_probes <= max_half_open_calls` to prevent thundering herds from overwhelming recovering providers.
|
|
255
|
+
- **`Retry-After` Compliance**: Automatically extracts and honors upstream rate-limit headers.
|
|
256
|
+
- Learn more in [Reliability Model](docs/RELIABILITY_MODEL.md).
|
|
257
|
+
|
|
258
|
+
### 2. Agent Continuation Protocol (ACP v1) & Durable State
|
|
259
|
+
Long-running agent workflows cannot depend on ephemeral memory:
|
|
260
|
+
- **Turn Checkpoints**: Preserves active context, token expenditure, and execution state in a local **SQLite WAL store**.
|
|
261
|
+
- **Operation Lifecycle Receipts**: Transitions each operation through `PREPARED` $\to$ `SUBMITTED` $\to$ `ACKNOWLEDGED` $\to$ `INDETERMINATE`.
|
|
262
|
+
- Learn more in [Agent Continuation Protocol](docs/AGENT_CONTINUATION_PROTOCOL.md) and [Durable State](docs/DURABLE_STATE.md).
|
|
263
|
+
|
|
264
|
+
### 3. Universal Protocol Intermediate Representation (IR)
|
|
265
|
+
Converts seamlessly between heterogeneous provider formats on failover:
|
|
266
|
+
- Canonical dataclasses: `NormalizedRequest`, `NormalizedMessage`, `NormalizedToolCall`, `NormalizedResponse`.
|
|
267
|
+
- Dynamic translation across Anthropic (`/v1/messages`), OpenAI (`/v1/chat/completions`), and Google Gemini.
|
|
268
|
+
- Preserves thinking signatures, tool definitions, and system prompts across migrations.
|
|
269
|
+
- Learn more in [Semantic Failover](docs/SEMANTIC_FAILOVER.md).
|
|
270
|
+
|
|
271
|
+
### 4. Idempotent Tool Execution Ledger
|
|
272
|
+
Prevents the catastrophic "double-spend" of autonomous coding agents:
|
|
273
|
+
- **Receipt Suppression**: Tool calls record their unique call ID and content hash. If an upstream drops after execution, the retry matches the committed receipt and serves cached output without re-executing.
|
|
274
|
+
- **Rule 3 Tool Safety**: Fails closed on missing required arguments. Repairs syntactic JSON/markdown fences but strictly forbids hallucinating or altering semantic arguments.
|
|
275
|
+
- Learn more in [Tool Safety & Idempotency](docs/TOOL_SAFETY.md).
|
|
276
|
+
|
|
277
|
+
### 5. Diagnostic Context Compaction (Rule 2)
|
|
278
|
+
When failing over to models with smaller context windows:
|
|
279
|
+
- **Preserves Critical Anchor Points**: Never truncates the initial system prompt or root user instructions.
|
|
280
|
+
- **Diagnostic Tool Extraction**: Instead of deleting tool results, replaces verbose build/lint/test logs with structured status lines (`[Exit 0: 42 files passed, 1 warning]`).
|
|
281
|
+
- Learn more in [Context Model](docs/CONTEXT_MODEL.md).
|
|
282
|
+
|
|
283
|
+
### 6. Calibrated Task Selection & Privacy Tiers
|
|
284
|
+
Intelligent candidate selection across multiple dimensions:
|
|
285
|
+
- **Data Privacy Profiles**: Strictly enforces routing policies (`AIR_GAPPED`, `LOCAL_ONLY`, `PUBLIC_ALLOWED`).
|
|
286
|
+
- **Independent Resource Lanes**: Keeps credential quotas isolated to prevent cross-pool starvation.
|
|
287
|
+
- **Confidence Calibration**: Adjusts selection probabilities based on historical empirical endpoint performance.
|
|
288
|
+
- Learn more in [Routing Policy](docs/ROUTING_POLICY.md).
|
|
289
|
+
|
|
290
|
+
---
|
|
291
|
+
|
|
292
|
+
## ๐ Empirical Benchmarks (B1โB15)
|
|
293
|
+
|
|
294
|
+
Evaluated across **15 deterministic stress scenarios** (permanent outages, 429 rate limits, timeouts, context overflows, malformed tool syntax, semantic schema violations, tool idempotency, mid-stream disconnects, provider cascades, pool isolation, cost ceilings, tool-reliability routing, and capability mismatches) against 6 in-process baseline architectures.
|
|
295
|
+
|
|
296
|
+
Results from official reproducible run (`results/2026-09-06-1943ba8/report.md`, 3 iterations, seed 42):
|
|
297
|
+
|
|
298
|
+
| System Architecture | Request Completion | Autonomous Recovery | Median Latency | P95 Latency | Semantic Error Rate |
|
|
299
|
+
|---|:---:|:---:|:---:|:---:|:---:|
|
|
300
|
+
| **โก LLM-Circuit-Breaker-V3** | **100.0%** | **80.0%** | **12.12 ms** | **313.64 ms** | **0.0%** |
|
|
301
|
+
| **Baseline-A (Direct Provider)** | 0.0% | 0.0% | 0.02 ms | 0.40 ms | 20.0% |
|
|
302
|
+
| **Baseline-B (Same-Provider Retry)** | 20.0% | 20.0% | 0.05 ms | 0.57 ms | 20.0% |
|
|
303
|
+
| **Baseline-C (Static Fallback)** | 33.3% | 33.3% | 0.03 ms | 0.32 ms | 20.0% |
|
|
304
|
+
| **Baseline-D (Breaker + Static Fallback)** | 33.3% | 33.3% | 0.04 ms | 0.29 ms | 20.0% |
|
|
305
|
+
| **Baseline-E (V1 Prototype Router)** | 53.3% | 53.3% | 0.13 ms | 5.07 ms | 20.0% |
|
|
306
|
+
| **Baseline-F (Standard Router Seam)** | 33.3% | 33.3% | 7.98 ms | 24.68 ms | 20.0% |
|
|
307
|
+
|
|
308
|
+
> **Key Takeaways**:
|
|
309
|
+
> 1. **Zero Semantic Errors**: LLM-Circuit-Breaker-V3 achieves 0.0% semantic error rate by strictly failing closed on invalid tool arguments (B6, B7, B14), whereas all baselines forward malformed tool calls that crash agent loops.
|
|
310
|
+
> 2. **100% Completion**: Only V3 survives context overflows (via diagnostic compaction) and rate limits (via sliding-window failover and `Retry-After` backoff).
|
|
311
|
+
> 3. **Reproduce Locally**: Run `python -m benchmarks.run` to execute the full test harness. Detailed methodology available in [docs/BENCHMARKS.md](docs/BENCHMARKS.md).
|
|
312
|
+
|
|
313
|
+
---
|
|
314
|
+
|
|
315
|
+
## ๐ฅ Architectural Comparison
|
|
316
|
+
|
|
317
|
+
How **LLM Circuit Breaker** compares to industry proxies and edge gateways:
|
|
318
|
+
|
|
319
|
+
| Architectural Dimension | **โก LLM Circuit Breaker** | **LiteLLM Proxy** | **Cloudflare AI Gateway** | **Portkey Gateway** | **OpenRouter** |
|
|
320
|
+
|---|:---:|:---:|:---:|:---:|:---:|
|
|
321
|
+
| **Circuit Breaker Engine** | **6-State FSM** with bounded half-open probe permits & sliding windows | Cooldown timer (`time + 60s`), no permit concurrency limits | Dynamic retry policy | Proprietary cloud breaker (enterprise tier) | Static upstream server retry |
|
|
322
|
+
| **Agent Tool Execution Ledger** | **Yes**: Tracks lifecycle receipts, prevents duplicate execution on retry | โ No: Blind replay on 5xx drops | โ No | โ No | โ No |
|
|
323
|
+
| **Context Compaction on Failover** | **Yes**: Diagnostic compaction preserves root goal + extracts exit codes | โ No: Naive head/tail truncation | โ No | โ No | โ No |
|
|
324
|
+
| **Universal Protocol IR** | **Yes**: Native cross-translation (Anthropic $\leftrightarrow$ OpenAI $\leftrightarrow$ Gemini) | Partial: In-memory dict remapping | โ No: Separate endpoints | Partial: Gateway REST wrappers | โ No: Standard OpenAI schema |
|
|
325
|
+
| **Streaming Safety** | **Interruption Boundary**: Prevents mid-stream model splicing | Splicing on failure | Aborts stream | Aborts stream | Aborts stream |
|
|
326
|
+
| **Deployment Footprint** | **Zero core dependencies**; self-contained Python package | Heavy dependencies (FastAPI, Prisma, Redis, Postgres) | Cloudflare Edge Worker (Cloud only) | SaaS cloud or enterprise container | Cloud-only API broker |
|
|
327
|
+
| **Privacy & Security** | **SSRF protection, CRLF sanitization, air-gapped routing tiers, local-first** | Cloud telemetry by default | Cloud control plane | Cloud control plane | Third-party proxy |
|
|
328
|
+
|
|
329
|
+
*See detailed technical comparisons in [docs/COMPETITOR_MATRIX.md](docs/COMPETITOR_MATRIX.md).*
|
|
330
|
+
|
|
331
|
+
---
|
|
332
|
+
|
|
333
|
+
## ๐ก๏ธ Security & Operational Hardening
|
|
334
|
+
|
|
335
|
+
LLM Circuit Breaker is built defensively for mission-critical self-hosted environments:
|
|
336
|
+
|
|
337
|
+
- **SSRF Defense**: Automatically blocks upstream URLs resolving to loopback (`127.0.0.1`) or RFC 1918 private subnets unless explicitly enabled via `LLM_BREAKER_ALLOW_LOCAL_UPSTREAM=1`. Cloud metadata endpoints (`169.254.169.254`) are **permanently refused**.
|
|
338
|
+
- **Payload Limits**: Rejects requests and responses exceeding 10 MB to prevent memory exhaustion attacks.
|
|
339
|
+
- **Credential Redaction**: Emits structured JSON events on the `durallm.events` logger with API keys and bearer tokens strictly masked.
|
|
340
|
+
- **Clean Environment Separation**: Zero network calls or file scans on import. Credential discovery is strictly opt-in.
|
|
341
|
+
|
|
342
|
+
---
|
|
343
|
+
|
|
344
|
+
## ๐ Documentation Hub
|
|
345
|
+
|
|
346
|
+
Explore in-depth design specifications, formal models, and operational runbooks:
|
|
347
|
+
|
|
348
|
+
- ๐๏ธ [Architecture Overview](ARCHITECTURE.md) โ System design, components, and dataflow.
|
|
349
|
+
- ๐ก๏ธ [Reliability & FSM Model](docs/RELIABILITY_MODEL.md) โ Formal 6-state FSM state transitions.
|
|
350
|
+
- ๐๏ธ [Failure Taxonomy](docs/FAILURE_TAXONOMY.md) โ Comprehensive classification of LLM failure modes.
|
|
351
|
+
- ๐ [Semantic Failover & Protocol IR](docs/SEMANTIC_FAILOVER.md) โ Cross-model payload translation.
|
|
352
|
+
- ๐ [Tool Safety & Idempotency Ledger](docs/TOOL_SAFETY.md) โ Replay suppression and schema validation.
|
|
353
|
+
- ๐๏ธ [Context Compaction Engine](docs/CONTEXT_MODEL.md) โ Hierarchical compaction preserving anchor instructions.
|
|
354
|
+
- ๐ [Streaming Architecture](docs/STREAMING.md) โ Native SSE pass-through and interruption boundaries.
|
|
355
|
+
- ๐ค [Agent Continuation Protocol (ACP v1)](docs/AGENT_CONTINUATION_PROTOCOL.md) โ Durable session recovery.
|
|
356
|
+
- ๐พ [Durable Persistence (SQLite WAL)](docs/DURABLE_STATE.md) โ Storage engine and ACID guarantees.
|
|
357
|
+
- ๐ฏ [Routing Policy & Calibration](docs/ROUTING_POLICY.md) โ Scorecards, privacy tiers, and resource lanes.
|
|
358
|
+
- ๐งช [Client Compatibility Matrix](docs/CLIENT_COMPATIBILITY.md) โ Recorded fixtures for Claude Code, Hermes, OpenClaw.
|
|
359
|
+
- ๐ [Benchmark Report (B1โB15)](docs/BENCHMARKS.md) โ Full methodology and empirical data.
|
|
360
|
+
- ๐ฅ [Competitor Deep-Dive](docs/COMPETITOR_MATRIX.md) โ Exhaustive feature-by-feature comparison.
|
|
361
|
+
- ๐ ๏ธ [Production Operations Runbook](docs/OPERATIONS.md) โ Deployment, health checks, and metrics.
|
|
362
|
+
- ๐ [Engineering Self-Critique](docs/FINAL_SELF_CRITIQUE.md) โ Rigorous adversarial audit and known limitations.
|
|
363
|
+
|
|
364
|
+
---
|
|
365
|
+
|
|
366
|
+
## ๐ License
|
|
367
|
+
|
|
368
|
+
Distributed under the **MIT License**. Engineered for resilience, determinism, and zero-compromise agent safety.
|