model-router-cli 1.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- model_router_cli-1.0.0/LICENSE +22 -0
- model_router_cli-1.0.0/PKG-INFO +343 -0
- model_router_cli-1.0.0/README.md +306 -0
- model_router_cli-1.0.0/backend/app/analytics/service.py +119 -0
- model_router_cli-1.0.0/backend/app/analyzer/analyzer.py +67 -0
- model_router_cli-1.0.0/backend/app/analyzer/heuristics.py +192 -0
- model_router_cli-1.0.0/backend/app/api/routes.py +589 -0
- model_router_cli-1.0.0/backend/app/budgets/manager.py +39 -0
- model_router_cli-1.0.0/backend/app/cli/main.py +287 -0
- model_router_cli-1.0.0/backend/app/config/settings.py +43 -0
- model_router_cli-1.0.0/backend/app/experiments/service.py +85 -0
- model_router_cli-1.0.0/backend/app/fallback/handler.py +105 -0
- model_router_cli-1.0.0/backend/app/models/schemas.py +127 -0
- model_router_cli-1.0.0/backend/app/observability/events.py +43 -0
- model_router_cli-1.0.0/backend/app/providers/base.py +46 -0
- model_router_cli-1.0.0/backend/app/providers/external_providers.py +321 -0
- model_router_cli-1.0.0/backend/app/providers/mock_provider.py +108 -0
- model_router_cli-1.0.0/backend/app/providers/ollama_provider.py +141 -0
- model_router_cli-1.0.0/backend/app/providers/registry.py +35 -0
- model_router_cli-1.0.0/backend/app/router/engine.py +150 -0
- model_router_cli-1.0.0/backend/app/router/rules_engine.py +73 -0
- model_router_cli-1.0.0/backend/app/router/scoring.py +154 -0
- model_router_cli-1.0.0/backend/app/static/assets/index-CQFztymk.js +63 -0
- model_router_cli-1.0.0/backend/app/static/assets/index-DWa3sE4Y.css +2 -0
- model_router_cli-1.0.0/backend/app/static/favicon.png +0 -0
- model_router_cli-1.0.0/backend/app/static/favicon.svg +1 -0
- model_router_cli-1.0.0/backend/app/static/icons.svg +24 -0
- model_router_cli-1.0.0/backend/app/static/index.html +17 -0
- model_router_cli-1.0.0/backend/app/static/logo.png +0 -0
- model_router_cli-1.0.0/backend/app/storage/database.py +366 -0
- model_router_cli-1.0.0/backend/app/storage/models.py +202 -0
- model_router_cli-1.0.0/backend/model_router_cli.egg-info/PKG-INFO +343 -0
- model_router_cli-1.0.0/backend/model_router_cli.egg-info/SOURCES.txt +41 -0
- model_router_cli-1.0.0/backend/model_router_cli.egg-info/dependency_links.txt +1 -0
- model_router_cli-1.0.0/backend/model_router_cli.egg-info/entry_points.txt +2 -0
- model_router_cli-1.0.0/backend/model_router_cli.egg-info/requires.txt +13 -0
- model_router_cli-1.0.0/backend/model_router_cli.egg-info/top_level.txt +2 -0
- model_router_cli-1.0.0/backend/tests/test_analyzer.py +41 -0
- model_router_cli-1.0.0/backend/tests/test_e2e.py +127 -0
- model_router_cli-1.0.0/backend/tests/test_providers.py +21 -0
- model_router_cli-1.0.0/backend/tests/test_router.py +78 -0
- model_router_cli-1.0.0/pyproject.toml +55 -0
- model_router_cli-1.0.0/setup.cfg +4 -0
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Model Router Contributors
|
|
4
|
+
Copyright (c) 2026 PicadoLabs
|
|
5
|
+
|
|
6
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
7
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
8
|
+
in the Software without restriction, including without limitation the rights
|
|
9
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
10
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
11
|
+
furnished to do so, subject to the following conditions:
|
|
12
|
+
|
|
13
|
+
The above copyright notice and this permission notice shall be included in all
|
|
14
|
+
copies or substantial portions of the Software.
|
|
15
|
+
|
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
17
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
18
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
19
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
20
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
21
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
22
|
+
SOFTWARE.
|
|
@@ -0,0 +1,343 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: model-router-cli
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: Intelligent, explainable LLM request routing platform & AI Traffic Control Room
|
|
5
|
+
Author-email: PicadoLabs <picadolabs@gmail.com>
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://picadolabs.me
|
|
8
|
+
Project-URL: Repository, https://github.com/PicadoLabs/ai-model-router
|
|
9
|
+
Project-URL: Issues, https://github.com/PicadoLabs/ai-model-router/issues
|
|
10
|
+
Keywords: llm,ai,router,fastapi,ollama,openai,cost-optimization
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
19
|
+
Classifier: Topic :: Software Development :: Libraries :: Application Frameworks
|
|
20
|
+
Requires-Python: >=3.10
|
|
21
|
+
Description-Content-Type: text/markdown
|
|
22
|
+
License-File: LICENSE
|
|
23
|
+
Requires-Dist: fastapi>=0.110.0
|
|
24
|
+
Requires-Dist: uvicorn[standard]>=0.28.0
|
|
25
|
+
Requires-Dist: pydantic>=2.6.0
|
|
26
|
+
Requires-Dist: pydantic-settings>=2.2.0
|
|
27
|
+
Requires-Dist: sqlalchemy>=2.0.28
|
|
28
|
+
Requires-Dist: aiosqlite>=0.20.0
|
|
29
|
+
Requires-Dist: httpx>=0.27.0
|
|
30
|
+
Requires-Dist: typer>=0.12.0
|
|
31
|
+
Requires-Dist: rich>=13.7.0
|
|
32
|
+
Requires-Dist: sse-starlette>=2.0.0
|
|
33
|
+
Requires-Dist: python-multipart>=0.0.9
|
|
34
|
+
Requires-Dist: python-dotenv>=1.0.1
|
|
35
|
+
Requires-Dist: tiktoken>=0.6.0
|
|
36
|
+
Dynamic: license-file
|
|
37
|
+
|
|
38
|
+
# Model Router
|
|
39
|
+
|
|
40
|
+
Intelligent, explainable, cost- and latency-aware LLM request routing platform and AI Traffic Control Room.
|
|
41
|
+
|
|
42
|
+
---
|
|
43
|
+
|
|
44
|
+
## Overview
|
|
45
|
+
|
|
46
|
+
Model Router intercepts incoming AI requests, analyzes their task type and continuous complexity, evaluates available models against a configurable multi-criteria scoring objective, selects the optimal candidate, and dispatches the request with automatic fallback handling and budget guards.
|
|
47
|
+
|
|
48
|
+
The platform is designed local-first, allowing full local development and testing using Mock models or local Ollama instances without requiring paid external API keys.
|
|
49
|
+
|
|
50
|
+
---
|
|
51
|
+
|
|
52
|
+
## Key Features
|
|
53
|
+
|
|
54
|
+
- **Dual-Mode Request Analyzer**: Deterministic heuristics (<3ms latency overhead) for 12 task types, continuous complexity scoring (0.05 to 0.99), and requirement detection, plus an optional LLM classifier mode.
|
|
55
|
+
- **Explainable Routing Engine**: Multi-criteria weighted scoring across Quality, Cost Efficiency, Speed, Capabilities, and Reliability with transparent decision factor reports and candidate rejection logs.
|
|
56
|
+
- **Provider Abstraction**: Decoupled adapters for Mock (simulation), Ollama (local), OpenAI, Anthropic, and Google Gemini.
|
|
57
|
+
- **Resilience and Tiered Fallback**: Automated retry classification for transient errors (timeouts, HTTP 429, 503) and tiered fallback to local/mock alternatives.
|
|
58
|
+
- **Budget Control Guards**: Real-time spend tracking with automated threshold interventions (80% cost optimization, 95% local-only saver, 100% block).
|
|
59
|
+
- **Traffic Control Room UI**: Real-time operational interface with seamless dark/light mode switching, featuring live topology graphs, playground inspector, SSE live request stream, telemetry export as CSV/JSON, visual rules builder, and cost savings simulator.
|
|
60
|
+
- **Developer CLI**: Terminal diagnostics (`doctor`), routing dry-run (`route`), execution (`run`), model catalog (`models`), and analytics (`analytics`).
|
|
61
|
+
|
|
62
|
+
---
|
|
63
|
+
|
|
64
|
+
## Architecture & Workflow
|
|
65
|
+
|
|
66
|
+
```text
|
|
67
|
+
[ Client / SDK / Typer CLI ]
|
|
68
|
+
│
|
|
69
|
+
▼
|
|
70
|
+
[ FastAPI Gateway (Port 8000) ]
|
|
71
|
+
│
|
|
72
|
+
┌────────┴──────────────────────────┐
|
|
73
|
+
│ 1. Request Analyzer (<3ms) │ --> Task Type, Complexity, Context Size
|
|
74
|
+
│ 2. Priority Rules Evaluation │ --> Conditional Overrides
|
|
75
|
+
│ 3. Candidate Hard Pruning │ --> Filter Ineligible Models (Context / Caps)
|
|
76
|
+
│ 4. Multi-Criteria Scoring │ --> Normalized 0-100 Score across 5 Dimensions
|
|
77
|
+
│ 5. Decision Factor Generator │ --> Itemized Explainability Breakdown
|
|
78
|
+
└────────┬──────────────────────────┘
|
|
79
|
+
│
|
|
80
|
+
▼
|
|
81
|
+
[ Fallback Supervisor & Provider Layer ]
|
|
82
|
+
├── Local: Ollama Provider (qwen2.5-coder, llama3.2, deepseek-r1)
|
|
83
|
+
├── Simulated: In-Memory Mock Provider (Zero Cost)
|
|
84
|
+
└── Cloud: OpenAI, Anthropic, Google Gemini (Optional)
|
|
85
|
+
│
|
|
86
|
+
▼
|
|
87
|
+
[ Storage & Observability Engine ]
|
|
88
|
+
├── Asynchronous SQLite WAL Database (`model_router.db`)
|
|
89
|
+
└── Server-Sent Events (SSE) Stream -> React Control Room (Port 5173)
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
---
|
|
93
|
+
|
|
94
|
+
## Supported Platforms & Prerequisites
|
|
95
|
+
|
|
96
|
+
### Supported Platforms
|
|
97
|
+
- Linux (Ubuntu 20.04+, Debian 11+, Fedora)
|
|
98
|
+
- macOS (macOS 12+ / Apple Silicon & Intel)
|
|
99
|
+
- Windows (Windows 10, Windows 11 / PowerShell & WSL2)
|
|
100
|
+
|
|
101
|
+
### Prerequisites
|
|
102
|
+
- Python 3.10, 3.11, or 3.12
|
|
103
|
+
- Node.js 18+ and npm
|
|
104
|
+
- (Optional) [Ollama](https://ollama.com/) for local model inference
|
|
105
|
+
|
|
106
|
+
---
|
|
107
|
+
|
|
108
|
+
## Installation & Setup
|
|
109
|
+
|
|
110
|
+
### 1. Clone the Repository
|
|
111
|
+
```bash
|
|
112
|
+
git clone https://github.com/PicadoLabs/AI-Model-Router.git
|
|
113
|
+
cd AI-Model-Router
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
### 2. Backend Installation
|
|
117
|
+
```bash
|
|
118
|
+
# Create and activate virtual environment
|
|
119
|
+
python -m venv venv
|
|
120
|
+
# On Linux/macOS:
|
|
121
|
+
source venv/bin/activate
|
|
122
|
+
# On Windows PowerShell:
|
|
123
|
+
.\venv\Scripts\Activate.ps1
|
|
124
|
+
|
|
125
|
+
# Install Python dependencies
|
|
126
|
+
pip install -r requirements.txt
|
|
127
|
+
|
|
128
|
+
# Create environment file from template
|
|
129
|
+
cp .env.example .env
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
### 3. Frontend Installation
|
|
133
|
+
```bash
|
|
134
|
+
cd frontend
|
|
135
|
+
npm install
|
|
136
|
+
cd ..
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
---
|
|
140
|
+
|
|
141
|
+
## Configuration & Environment Variables
|
|
142
|
+
|
|
143
|
+
Configuration is loaded via Pydantic Settings from the `.env` file:
|
|
144
|
+
|
|
145
|
+
| Variable | Default | Description |
|
|
146
|
+
| :--- | :--- | :--- |
|
|
147
|
+
| `APP_ENV` | `development` | Application environment (`development`, `production`, `test`) |
|
|
148
|
+
| `PORT` | `8000` | FastAPI server port |
|
|
149
|
+
| `HOST` | `0.0.0.0` | FastAPI server host |
|
|
150
|
+
| `DATABASE_URL` | `sqlite+aiosqlite:///./model_router.db` | SQLAlchemy database connection URI |
|
|
151
|
+
| `ROUTER_ANALYZER` | `rules` | Default analyzer mode (`rules` for heuristics, `llm` for model classifier) |
|
|
152
|
+
| `DEFAULT_ROUTING_POLICY` | `balanced` | Default routing weights (`balanced`, `lowest_cost`, `lowest_latency`, `highest_quality`) |
|
|
153
|
+
| `BASELINE_MODEL_ID` | `mock-power` | Reference model ID for calculating baseline cost savings |
|
|
154
|
+
| `DEFAULT_PROVIDER` | `mock` | Default execution provider (`mock`, `ollama`) |
|
|
155
|
+
| `OLLAMA_BASE_URL` | `http://localhost:11434` | Ollama HTTP endpoint |
|
|
156
|
+
| `OPENAI_API_KEY` | *(empty)* | Optional OpenAI API Key |
|
|
157
|
+
| `ANTHROPIC_API_KEY` | *(empty)* | Optional Anthropic API Key |
|
|
158
|
+
| `GEMINI_API_KEY` | *(empty)* | Optional Google Gemini API Key |
|
|
159
|
+
| `DAILY_BUDGET` | `10.00` | Daily spend limit in USD |
|
|
160
|
+
| `MONTHLY_BUDGET` | `100.00` | Monthly spend limit in USD |
|
|
161
|
+
| `MAX_RETRIES` | `2` | Maximum retries before triggering cascading fallback |
|
|
162
|
+
| `PROVIDER_TIMEOUT_SECONDS` | `30.0` | Provider HTTP timeout in seconds |
|
|
163
|
+
|
|
164
|
+
---
|
|
165
|
+
|
|
166
|
+
## Quickstart
|
|
167
|
+
|
|
168
|
+
### 1. Run System Diagnostics
|
|
169
|
+
```bash
|
|
170
|
+
python backend/app/cli/main.py doctor
|
|
171
|
+
```
|
|
172
|
+
|
|
173
|
+
### 2. Start the Backend API Server
|
|
174
|
+
```bash
|
|
175
|
+
python backend/main.py
|
|
176
|
+
# API server running at http://127.0.0.1:8000
|
|
177
|
+
# Interactive API docs available at http://127.0.0.1:8000/docs
|
|
178
|
+
```
|
|
179
|
+
|
|
180
|
+
### 3. Start the Control Room UI
|
|
181
|
+
In a separate terminal:
|
|
182
|
+
```bash
|
|
183
|
+
cd frontend
|
|
184
|
+
npm run dev
|
|
185
|
+
# Access UI at http://localhost:5173
|
|
186
|
+
```
|
|
187
|
+
|
|
188
|
+
---
|
|
189
|
+
|
|
190
|
+
## CLI Usage
|
|
191
|
+
|
|
192
|
+
The built-in Typer CLI provides terminal commands for inspection, diagnostics, and testing:
|
|
193
|
+
|
|
194
|
+
```bash
|
|
195
|
+
# Run system diagnostics & provider health checks
|
|
196
|
+
python backend/app/cli/main.py doctor
|
|
197
|
+
|
|
198
|
+
# Inspect routing decision for a prompt without executing (Dry Run)
|
|
199
|
+
python backend/app/cli/main.py route "Write a Python function to parse JSON"
|
|
200
|
+
|
|
201
|
+
# Route and execute a query through the selected model
|
|
202
|
+
python backend/app/cli/main.py run "Debug this distributed async deadlock in worker pool"
|
|
203
|
+
|
|
204
|
+
# List all registered models in the catalog
|
|
205
|
+
python backend/app/cli/main.py models
|
|
206
|
+
|
|
207
|
+
# View system-wide routing performance and cost savings analytics
|
|
208
|
+
python backend/app/cli/main.py analytics
|
|
209
|
+
```
|
|
210
|
+
|
|
211
|
+
---
|
|
212
|
+
|
|
213
|
+
## REST API Usage
|
|
214
|
+
|
|
215
|
+
### 1. Dry-Run Routing (`POST /api/route`)
|
|
216
|
+
```bash
|
|
217
|
+
curl -X POST http://127.0.0.1:8000/api/route \
|
|
218
|
+
-H "Content-Type: application/json" \
|
|
219
|
+
-d '{"prompt": "Write a quicksort algorithm in Python", "policy": "balanced"}'
|
|
220
|
+
```
|
|
221
|
+
|
|
222
|
+
### 2. End-to-End Routed Generation (`POST /api/generate`)
|
|
223
|
+
```bash
|
|
224
|
+
curl -X POST http://127.0.0.1:8000/api/generate \
|
|
225
|
+
-H "Content-Type: application/json" \
|
|
226
|
+
-d '{"prompt": "Explain the difference between TCP and UDP", "policy": "lowest_cost"}'
|
|
227
|
+
```
|
|
228
|
+
|
|
229
|
+
### 3. Fetch Registered Models (`GET /api/models`)
|
|
230
|
+
```bash
|
|
231
|
+
curl http://127.0.0.1:8000/api/models
|
|
232
|
+
```
|
|
233
|
+
|
|
234
|
+
### 4. Export Historical Traffic (`GET /api/traffic/export`)
|
|
235
|
+
Export all persisted traffic records for auditing, accounting, or latency analysis:
|
|
236
|
+
|
|
237
|
+
```bash
|
|
238
|
+
# Export as JSON
|
|
239
|
+
curl -OJ "http://127.0.0.1:8000/api/traffic/export?format=json"
|
|
240
|
+
|
|
241
|
+
# Export as CSV
|
|
242
|
+
curl -OJ "http://127.0.0.1:8000/api/traffic/export?format=csv"
|
|
243
|
+
```
|
|
244
|
+
|
|
245
|
+
Each export includes the timestamp, request ID, prompt preview, task type, complexity, selected model, input/output/total tokens, cost saved, and total latency. The `format` query parameter accepts only `csv` or `json`.
|
|
246
|
+
|
|
247
|
+
The same export is available in the frontend under **Traffic**. Select `CSV` or `JSON` beside **Export Telemetry**, then click the button to download the complete historical traffic dataset.
|
|
248
|
+
|
|
249
|
+
---
|
|
250
|
+
|
|
251
|
+
## Running Tests
|
|
252
|
+
|
|
253
|
+
The test suite includes 18 automated unit, integration, and end-to-end tests covering prompt heuristics, candidate pruning, scoring weights, provider execution, error fallbacks, REST endpoints, and CSV/JSON traffic exports:
|
|
254
|
+
|
|
255
|
+
```bash
|
|
256
|
+
# Run the backend test suite from the repository root
|
|
257
|
+
pytest backend/tests
|
|
258
|
+
|
|
259
|
+
# Or run it from the backend directory
|
|
260
|
+
cd backend
|
|
261
|
+
python -m pytest tests
|
|
262
|
+
|
|
263
|
+
# Run frontend production build test
|
|
264
|
+
cd frontend
|
|
265
|
+
npm run build
|
|
266
|
+
```
|
|
267
|
+
|
|
268
|
+
---
|
|
269
|
+
|
|
270
|
+
## Project Structure
|
|
271
|
+
|
|
272
|
+
```text
|
|
273
|
+
AI-Model-Router/
|
|
274
|
+
├── .github/
|
|
275
|
+
│ ├── ISSUE_TEMPLATE/
|
|
276
|
+
│ │ ├── bug_report.md
|
|
277
|
+
│ │ └── feature_request.md
|
|
278
|
+
│ ├── pull_request_template.md
|
|
279
|
+
│ └── workflows/
|
|
280
|
+
│ └── ci.yml
|
|
281
|
+
├── backend/
|
|
282
|
+
│ ├── app/
|
|
283
|
+
│ │ ├── analytics/ # Cost savings and aggregate analytics service
|
|
284
|
+
│ │ ├── analyzer/ # Dual-mode request analyzer (heuristics & LLM)
|
|
285
|
+
│ │ ├── api/ # FastAPI REST endpoints and request handlers
|
|
286
|
+
│ │ ├── budgets/ # Spend tracking and automated threshold manager
|
|
287
|
+
│ │ ├── cli/ # Typer CLI application (doctor, route, run, etc.)
|
|
288
|
+
│ │ ├── config/ # Pydantic Settings environment configuration
|
|
289
|
+
│ │ ├── experiments/ # A/B policy experimentation service
|
|
290
|
+
│ │ ├── fallback/ # Error classifier and tiered fallback supervisor
|
|
291
|
+
│ │ ├── models/ # Pydantic schemas (RequestAnalysis, RoutingDecision)
|
|
292
|
+
│ │ ├── observability/ # Redacted structured JSON event logger
|
|
293
|
+
│ │ ├── providers/ # Decoupled adapters (Mock, Ollama, Cloud)
|
|
294
|
+
│ │ ├── router/ # Core scoring matrix, pruner, and rules engine
|
|
295
|
+
│ │ └── storage/ # SQLAlchemy models and SQLite async database
|
|
296
|
+
│ ├── tests/ # Pytest test suite (15 passing tests)
|
|
297
|
+
│ ├── main.py # FastAPI application entrypoint
|
|
298
|
+
│ └── requirements.txt # Python backend dependencies
|
|
299
|
+
├── frontend/
|
|
300
|
+
│ ├── src/
|
|
301
|
+
│ │ ├── components/ # UI components (Navbar, RoutingMap topology graph)
|
|
302
|
+
│ │ ├── pages/ # Control Room pages (Dashboard, Playground, Rules, etc.)
|
|
303
|
+
│ │ └── types/ # TypeScript data interfaces
|
|
304
|
+
│ └── package.json # Node.js dependencies
|
|
305
|
+
├── .env.example # Configuration template
|
|
306
|
+
├── .gitignore # Git exclusions
|
|
307
|
+
├── CODE_OF_CONDUCT.md # Contributor Covenant Code of Conduct
|
|
308
|
+
├── CONTRIBUTING.md # Contribution guidelines and workflow
|
|
309
|
+
├── LICENSE # MIT License
|
|
310
|
+
├── README.md # Project documentation
|
|
311
|
+
├── requirements.txt # Root Python dependencies
|
|
312
|
+
└── SECURITY.md # Vulnerability reporting and security policy
|
|
313
|
+
```
|
|
314
|
+
|
|
315
|
+
---
|
|
316
|
+
|
|
317
|
+
## Contributing
|
|
318
|
+
|
|
319
|
+
We welcome contributions from the community. Please review [CONTRIBUTING.md](CONTRIBUTING.md) for details on our development setup, coding standards, branch conventions, and pull request process.
|
|
320
|
+
|
|
321
|
+
Please note that this project is released with a [Code of Conduct](CODE_OF_CONDUCT.md). By participating in this project you agree to abide by its terms.
|
|
322
|
+
|
|
323
|
+
---
|
|
324
|
+
|
|
325
|
+
## Security
|
|
326
|
+
|
|
327
|
+
Security and privacy are core to Model Router. For vulnerability reporting procedures and our zero-secret-exposure policy, please refer to [SECURITY.md](SECURITY.md).
|
|
328
|
+
|
|
329
|
+
---
|
|
330
|
+
|
|
331
|
+
## License
|
|
332
|
+
|
|
333
|
+
This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.
|
|
334
|
+
|
|
335
|
+
---
|
|
336
|
+
|
|
337
|
+
## PicadoLabs
|
|
338
|
+
|
|
339
|
+
Maintained and architected by **PicadoLabs**.
|
|
340
|
+
|
|
341
|
+
- **Organization**: [PicadoLabs](https://github.com/PicadoLabs)
|
|
342
|
+
- **Website**: [https://picadolabs.me](https://picadolabs.me)
|
|
343
|
+
- **Contact**: [picadolabs@gmail.com](mailto:picadolabs@gmail.com)
|
|
@@ -0,0 +1,306 @@
|
|
|
1
|
+
# Model Router
|
|
2
|
+
|
|
3
|
+
Intelligent, explainable, cost- and latency-aware LLM request routing platform and AI Traffic Control Room.
|
|
4
|
+
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
## Overview
|
|
8
|
+
|
|
9
|
+
Model Router intercepts incoming AI requests, analyzes their task type and continuous complexity, evaluates available models against a configurable multi-criteria scoring objective, selects the optimal candidate, and dispatches the request with automatic fallback handling and budget guards.
|
|
10
|
+
|
|
11
|
+
The platform is designed local-first, allowing full local development and testing using Mock models or local Ollama instances without requiring paid external API keys.
|
|
12
|
+
|
|
13
|
+
---
|
|
14
|
+
|
|
15
|
+
## Key Features
|
|
16
|
+
|
|
17
|
+
- **Dual-Mode Request Analyzer**: Deterministic heuristics (<3ms latency overhead) for 12 task types, continuous complexity scoring (0.05 to 0.99), and requirement detection, plus an optional LLM classifier mode.
|
|
18
|
+
- **Explainable Routing Engine**: Multi-criteria weighted scoring across Quality, Cost Efficiency, Speed, Capabilities, and Reliability with transparent decision factor reports and candidate rejection logs.
|
|
19
|
+
- **Provider Abstraction**: Decoupled adapters for Mock (simulation), Ollama (local), OpenAI, Anthropic, and Google Gemini.
|
|
20
|
+
- **Resilience and Tiered Fallback**: Automated retry classification for transient errors (timeouts, HTTP 429, 503) and tiered fallback to local/mock alternatives.
|
|
21
|
+
- **Budget Control Guards**: Real-time spend tracking with automated threshold interventions (80% cost optimization, 95% local-only saver, 100% block).
|
|
22
|
+
- **Traffic Control Room UI**: Real-time operational interface with seamless dark/light mode switching, featuring live topology graphs, playground inspector, SSE live request stream, telemetry export as CSV/JSON, visual rules builder, and cost savings simulator.
|
|
23
|
+
- **Developer CLI**: Terminal diagnostics (`doctor`), routing dry-run (`route`), execution (`run`), model catalog (`models`), and analytics (`analytics`).
|
|
24
|
+
|
|
25
|
+
---
|
|
26
|
+
|
|
27
|
+
## Architecture & Workflow
|
|
28
|
+
|
|
29
|
+
```text
|
|
30
|
+
[ Client / SDK / Typer CLI ]
|
|
31
|
+
│
|
|
32
|
+
▼
|
|
33
|
+
[ FastAPI Gateway (Port 8000) ]
|
|
34
|
+
│
|
|
35
|
+
┌────────┴──────────────────────────┐
|
|
36
|
+
│ 1. Request Analyzer (<3ms) │ --> Task Type, Complexity, Context Size
|
|
37
|
+
│ 2. Priority Rules Evaluation │ --> Conditional Overrides
|
|
38
|
+
│ 3. Candidate Hard Pruning │ --> Filter Ineligible Models (Context / Caps)
|
|
39
|
+
│ 4. Multi-Criteria Scoring │ --> Normalized 0-100 Score across 5 Dimensions
|
|
40
|
+
│ 5. Decision Factor Generator │ --> Itemized Explainability Breakdown
|
|
41
|
+
└────────┬──────────────────────────┘
|
|
42
|
+
│
|
|
43
|
+
▼
|
|
44
|
+
[ Fallback Supervisor & Provider Layer ]
|
|
45
|
+
├── Local: Ollama Provider (qwen2.5-coder, llama3.2, deepseek-r1)
|
|
46
|
+
├── Simulated: In-Memory Mock Provider (Zero Cost)
|
|
47
|
+
└── Cloud: OpenAI, Anthropic, Google Gemini (Optional)
|
|
48
|
+
│
|
|
49
|
+
▼
|
|
50
|
+
[ Storage & Observability Engine ]
|
|
51
|
+
├── Asynchronous SQLite WAL Database (`model_router.db`)
|
|
52
|
+
└── Server-Sent Events (SSE) Stream -> React Control Room (Port 5173)
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
---
|
|
56
|
+
|
|
57
|
+
## Supported Platforms & Prerequisites
|
|
58
|
+
|
|
59
|
+
### Supported Platforms
|
|
60
|
+
- Linux (Ubuntu 20.04+, Debian 11+, Fedora)
|
|
61
|
+
- macOS (macOS 12+ / Apple Silicon & Intel)
|
|
62
|
+
- Windows (Windows 10, Windows 11 / PowerShell & WSL2)
|
|
63
|
+
|
|
64
|
+
### Prerequisites
|
|
65
|
+
- Python 3.10, 3.11, or 3.12
|
|
66
|
+
- Node.js 18+ and npm
|
|
67
|
+
- (Optional) [Ollama](https://ollama.com/) for local model inference
|
|
68
|
+
|
|
69
|
+
---
|
|
70
|
+
|
|
71
|
+
## Installation & Setup
|
|
72
|
+
|
|
73
|
+
### 1. Clone the Repository
|
|
74
|
+
```bash
|
|
75
|
+
git clone https://github.com/PicadoLabs/AI-Model-Router.git
|
|
76
|
+
cd AI-Model-Router
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
### 2. Backend Installation
|
|
80
|
+
```bash
|
|
81
|
+
# Create and activate virtual environment
|
|
82
|
+
python -m venv venv
|
|
83
|
+
# On Linux/macOS:
|
|
84
|
+
source venv/bin/activate
|
|
85
|
+
# On Windows PowerShell:
|
|
86
|
+
.\venv\Scripts\Activate.ps1
|
|
87
|
+
|
|
88
|
+
# Install Python dependencies
|
|
89
|
+
pip install -r requirements.txt
|
|
90
|
+
|
|
91
|
+
# Create environment file from template
|
|
92
|
+
cp .env.example .env
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
### 3. Frontend Installation
|
|
96
|
+
```bash
|
|
97
|
+
cd frontend
|
|
98
|
+
npm install
|
|
99
|
+
cd ..
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
---
|
|
103
|
+
|
|
104
|
+
## Configuration & Environment Variables
|
|
105
|
+
|
|
106
|
+
Configuration is loaded via Pydantic Settings from the `.env` file:
|
|
107
|
+
|
|
108
|
+
| Variable | Default | Description |
|
|
109
|
+
| :--- | :--- | :--- |
|
|
110
|
+
| `APP_ENV` | `development` | Application environment (`development`, `production`, `test`) |
|
|
111
|
+
| `PORT` | `8000` | FastAPI server port |
|
|
112
|
+
| `HOST` | `0.0.0.0` | FastAPI server host |
|
|
113
|
+
| `DATABASE_URL` | `sqlite+aiosqlite:///./model_router.db` | SQLAlchemy database connection URI |
|
|
114
|
+
| `ROUTER_ANALYZER` | `rules` | Default analyzer mode (`rules` for heuristics, `llm` for model classifier) |
|
|
115
|
+
| `DEFAULT_ROUTING_POLICY` | `balanced` | Default routing weights (`balanced`, `lowest_cost`, `lowest_latency`, `highest_quality`) |
|
|
116
|
+
| `BASELINE_MODEL_ID` | `mock-power` | Reference model ID for calculating baseline cost savings |
|
|
117
|
+
| `DEFAULT_PROVIDER` | `mock` | Default execution provider (`mock`, `ollama`) |
|
|
118
|
+
| `OLLAMA_BASE_URL` | `http://localhost:11434` | Ollama HTTP endpoint |
|
|
119
|
+
| `OPENAI_API_KEY` | *(empty)* | Optional OpenAI API Key |
|
|
120
|
+
| `ANTHROPIC_API_KEY` | *(empty)* | Optional Anthropic API Key |
|
|
121
|
+
| `GEMINI_API_KEY` | *(empty)* | Optional Google Gemini API Key |
|
|
122
|
+
| `DAILY_BUDGET` | `10.00` | Daily spend limit in USD |
|
|
123
|
+
| `MONTHLY_BUDGET` | `100.00` | Monthly spend limit in USD |
|
|
124
|
+
| `MAX_RETRIES` | `2` | Maximum retries before triggering cascading fallback |
|
|
125
|
+
| `PROVIDER_TIMEOUT_SECONDS` | `30.0` | Provider HTTP timeout in seconds |
|
|
126
|
+
|
|
127
|
+
---
|
|
128
|
+
|
|
129
|
+
## Quickstart
|
|
130
|
+
|
|
131
|
+
### 1. Run System Diagnostics
|
|
132
|
+
```bash
|
|
133
|
+
python backend/app/cli/main.py doctor
|
|
134
|
+
```
|
|
135
|
+
|
|
136
|
+
### 2. Start the Backend API Server
|
|
137
|
+
```bash
|
|
138
|
+
python backend/main.py
|
|
139
|
+
# API server running at http://127.0.0.1:8000
|
|
140
|
+
# Interactive API docs available at http://127.0.0.1:8000/docs
|
|
141
|
+
```
|
|
142
|
+
|
|
143
|
+
### 3. Start the Control Room UI
|
|
144
|
+
In a separate terminal:
|
|
145
|
+
```bash
|
|
146
|
+
cd frontend
|
|
147
|
+
npm run dev
|
|
148
|
+
# Access UI at http://localhost:5173
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
---
|
|
152
|
+
|
|
153
|
+
## CLI Usage
|
|
154
|
+
|
|
155
|
+
The built-in Typer CLI provides terminal commands for inspection, diagnostics, and testing:
|
|
156
|
+
|
|
157
|
+
```bash
|
|
158
|
+
# Run system diagnostics & provider health checks
|
|
159
|
+
python backend/app/cli/main.py doctor
|
|
160
|
+
|
|
161
|
+
# Inspect routing decision for a prompt without executing (Dry Run)
|
|
162
|
+
python backend/app/cli/main.py route "Write a Python function to parse JSON"
|
|
163
|
+
|
|
164
|
+
# Route and execute a query through the selected model
|
|
165
|
+
python backend/app/cli/main.py run "Debug this distributed async deadlock in worker pool"
|
|
166
|
+
|
|
167
|
+
# List all registered models in the catalog
|
|
168
|
+
python backend/app/cli/main.py models
|
|
169
|
+
|
|
170
|
+
# View system-wide routing performance and cost savings analytics
|
|
171
|
+
python backend/app/cli/main.py analytics
|
|
172
|
+
```
|
|
173
|
+
|
|
174
|
+
---
|
|
175
|
+
|
|
176
|
+
## REST API Usage
|
|
177
|
+
|
|
178
|
+
### 1. Dry-Run Routing (`POST /api/route`)
|
|
179
|
+
```bash
|
|
180
|
+
curl -X POST http://127.0.0.1:8000/api/route \
|
|
181
|
+
-H "Content-Type: application/json" \
|
|
182
|
+
-d '{"prompt": "Write a quicksort algorithm in Python", "policy": "balanced"}'
|
|
183
|
+
```
|
|
184
|
+
|
|
185
|
+
### 2. End-to-End Routed Generation (`POST /api/generate`)
|
|
186
|
+
```bash
|
|
187
|
+
curl -X POST http://127.0.0.1:8000/api/generate \
|
|
188
|
+
-H "Content-Type: application/json" \
|
|
189
|
+
-d '{"prompt": "Explain the difference between TCP and UDP", "policy": "lowest_cost"}'
|
|
190
|
+
```
|
|
191
|
+
|
|
192
|
+
### 3. Fetch Registered Models (`GET /api/models`)
|
|
193
|
+
```bash
|
|
194
|
+
curl http://127.0.0.1:8000/api/models
|
|
195
|
+
```
|
|
196
|
+
|
|
197
|
+
### 4. Export Historical Traffic (`GET /api/traffic/export`)
|
|
198
|
+
Export all persisted traffic records for auditing, accounting, or latency analysis:
|
|
199
|
+
|
|
200
|
+
```bash
|
|
201
|
+
# Export as JSON
|
|
202
|
+
curl -OJ "http://127.0.0.1:8000/api/traffic/export?format=json"
|
|
203
|
+
|
|
204
|
+
# Export as CSV
|
|
205
|
+
curl -OJ "http://127.0.0.1:8000/api/traffic/export?format=csv"
|
|
206
|
+
```
|
|
207
|
+
|
|
208
|
+
Each export includes the timestamp, request ID, prompt preview, task type, complexity, selected model, input/output/total tokens, cost saved, and total latency. The `format` query parameter accepts only `csv` or `json`.
|
|
209
|
+
|
|
210
|
+
The same export is available in the frontend under **Traffic**. Select `CSV` or `JSON` beside **Export Telemetry**, then click the button to download the complete historical traffic dataset.
|
|
211
|
+
|
|
212
|
+
---
|
|
213
|
+
|
|
214
|
+
## Running Tests
|
|
215
|
+
|
|
216
|
+
The test suite includes 18 automated unit, integration, and end-to-end tests covering prompt heuristics, candidate pruning, scoring weights, provider execution, error fallbacks, REST endpoints, and CSV/JSON traffic exports:
|
|
217
|
+
|
|
218
|
+
```bash
|
|
219
|
+
# Run the backend test suite from the repository root
|
|
220
|
+
pytest backend/tests
|
|
221
|
+
|
|
222
|
+
# Or run it from the backend directory
|
|
223
|
+
cd backend
|
|
224
|
+
python -m pytest tests
|
|
225
|
+
|
|
226
|
+
# Run frontend production build test
|
|
227
|
+
cd frontend
|
|
228
|
+
npm run build
|
|
229
|
+
```
|
|
230
|
+
|
|
231
|
+
---
|
|
232
|
+
|
|
233
|
+
## Project Structure
|
|
234
|
+
|
|
235
|
+
```text
|
|
236
|
+
AI-Model-Router/
|
|
237
|
+
├── .github/
|
|
238
|
+
│ ├── ISSUE_TEMPLATE/
|
|
239
|
+
│ │ ├── bug_report.md
|
|
240
|
+
│ │ └── feature_request.md
|
|
241
|
+
│ ├── pull_request_template.md
|
|
242
|
+
│ └── workflows/
|
|
243
|
+
│ └── ci.yml
|
|
244
|
+
├── backend/
|
|
245
|
+
│ ├── app/
|
|
246
|
+
│ │ ├── analytics/ # Cost savings and aggregate analytics service
|
|
247
|
+
│ │ ├── analyzer/ # Dual-mode request analyzer (heuristics & LLM)
|
|
248
|
+
│ │ ├── api/ # FastAPI REST endpoints and request handlers
|
|
249
|
+
│ │ ├── budgets/ # Spend tracking and automated threshold manager
|
|
250
|
+
│ │ ├── cli/ # Typer CLI application (doctor, route, run, etc.)
|
|
251
|
+
│ │ ├── config/ # Pydantic Settings environment configuration
|
|
252
|
+
│ │ ├── experiments/ # A/B policy experimentation service
|
|
253
|
+
│ │ ├── fallback/ # Error classifier and tiered fallback supervisor
|
|
254
|
+
│ │ ├── models/ # Pydantic schemas (RequestAnalysis, RoutingDecision)
|
|
255
|
+
│ │ ├── observability/ # Redacted structured JSON event logger
|
|
256
|
+
│ │ ├── providers/ # Decoupled adapters (Mock, Ollama, Cloud)
|
|
257
|
+
│ │ ├── router/ # Core scoring matrix, pruner, and rules engine
|
|
258
|
+
│ │ └── storage/ # SQLAlchemy models and SQLite async database
|
|
259
|
+
│ ├── tests/ # Pytest test suite (15 passing tests)
|
|
260
|
+
│ ├── main.py # FastAPI application entrypoint
|
|
261
|
+
│ └── requirements.txt # Python backend dependencies
|
|
262
|
+
├── frontend/
|
|
263
|
+
│ ├── src/
|
|
264
|
+
│ │ ├── components/ # UI components (Navbar, RoutingMap topology graph)
|
|
265
|
+
│ │ ├── pages/ # Control Room pages (Dashboard, Playground, Rules, etc.)
|
|
266
|
+
│ │ └── types/ # TypeScript data interfaces
|
|
267
|
+
│ └── package.json # Node.js dependencies
|
|
268
|
+
├── .env.example # Configuration template
|
|
269
|
+
├── .gitignore # Git exclusions
|
|
270
|
+
├── CODE_OF_CONDUCT.md # Contributor Covenant Code of Conduct
|
|
271
|
+
├── CONTRIBUTING.md # Contribution guidelines and workflow
|
|
272
|
+
├── LICENSE # MIT License
|
|
273
|
+
├── README.md # Project documentation
|
|
274
|
+
├── requirements.txt # Root Python dependencies
|
|
275
|
+
└── SECURITY.md # Vulnerability reporting and security policy
|
|
276
|
+
```
|
|
277
|
+
|
|
278
|
+
---
|
|
279
|
+
|
|
280
|
+
## Contributing
|
|
281
|
+
|
|
282
|
+
We welcome contributions from the community. Please review [CONTRIBUTING.md](CONTRIBUTING.md) for details on our development setup, coding standards, branch conventions, and pull request process.
|
|
283
|
+
|
|
284
|
+
Please note that this project is released with a [Code of Conduct](CODE_OF_CONDUCT.md). By participating in this project you agree to abide by its terms.
|
|
285
|
+
|
|
286
|
+
---
|
|
287
|
+
|
|
288
|
+
## Security
|
|
289
|
+
|
|
290
|
+
Security and privacy are core to Model Router. For vulnerability reporting procedures and our zero-secret-exposure policy, please refer to [SECURITY.md](SECURITY.md).
|
|
291
|
+
|
|
292
|
+
---
|
|
293
|
+
|
|
294
|
+
## License
|
|
295
|
+
|
|
296
|
+
This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.
|
|
297
|
+
|
|
298
|
+
---
|
|
299
|
+
|
|
300
|
+
## PicadoLabs
|
|
301
|
+
|
|
302
|
+
Maintained and architected by **PicadoLabs**.
|
|
303
|
+
|
|
304
|
+
- **Organization**: [PicadoLabs](https://github.com/PicadoLabs)
|
|
305
|
+
- **Website**: [https://picadolabs.me](https://picadolabs.me)
|
|
306
|
+
- **Contact**: [picadolabs@gmail.com](mailto:picadolabs@gmail.com)
|