smartify-ai 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- smartify_ai-0.1.0/.gitignore +34 -0
- smartify_ai-0.1.0/DESIGN.md +389 -0
- smartify_ai-0.1.0/GAPS.md +59 -0
- smartify_ai-0.1.0/LAUNCH_PLAN.md +144 -0
- smartify_ai-0.1.0/LICENSE +21 -0
- smartify_ai-0.1.0/PKG-INFO +201 -0
- smartify_ai-0.1.0/README.md +163 -0
- smartify_ai-0.1.0/examples/calculator.yaml +134 -0
- smartify_ai-0.1.0/examples/file-creator.yaml +30 -0
- smartify_ai-0.1.0/examples/mcp-filesystem.yaml +44 -0
- smartify_ai-0.1.0/examples/minimal.yaml +20 -0
- smartify_ai-0.1.0/examples/parallel-swarm.yaml +146 -0
- smartify_ai-0.1.0/pyproject.toml +78 -0
- smartify_ai-0.1.0/src/smartify/__init__.py +3 -0
- smartify_ai-0.1.0/src/smartify/agents/__init__.py +0 -0
- smartify_ai-0.1.0/src/smartify/agents/adapters/__init__.py +13 -0
- smartify_ai-0.1.0/src/smartify/agents/adapters/anthropic.py +253 -0
- smartify_ai-0.1.0/src/smartify/agents/adapters/openai.py +289 -0
- smartify_ai-0.1.0/src/smartify/api/__init__.py +26 -0
- smartify_ai-0.1.0/src/smartify/api/auth.py +352 -0
- smartify_ai-0.1.0/src/smartify/api/errors.py +380 -0
- smartify_ai-0.1.0/src/smartify/api/events.py +345 -0
- smartify_ai-0.1.0/src/smartify/api/server.py +992 -0
- smartify_ai-0.1.0/src/smartify/cli/__init__.py +1 -0
- smartify_ai-0.1.0/src/smartify/cli/main.py +430 -0
- smartify_ai-0.1.0/src/smartify/engine/__init__.py +64 -0
- smartify_ai-0.1.0/src/smartify/engine/approval.py +479 -0
- smartify_ai-0.1.0/src/smartify/engine/orchestrator.py +1365 -0
- smartify_ai-0.1.0/src/smartify/engine/scheduler.py +380 -0
- smartify_ai-0.1.0/src/smartify/engine/spark.py +294 -0
- smartify_ai-0.1.0/src/smartify/guardrails/__init__.py +22 -0
- smartify_ai-0.1.0/src/smartify/guardrails/breakers.py +409 -0
- smartify_ai-0.1.0/src/smartify/models/__init__.py +61 -0
- smartify_ai-0.1.0/src/smartify/models/grid.py +625 -0
- smartify_ai-0.1.0/src/smartify/notifications/__init__.py +22 -0
- smartify_ai-0.1.0/src/smartify/notifications/webhook.py +556 -0
- smartify_ai-0.1.0/src/smartify/state/__init__.py +46 -0
- smartify_ai-0.1.0/src/smartify/state/checkpoint.py +558 -0
- smartify_ai-0.1.0/src/smartify/state/resume.py +301 -0
- smartify_ai-0.1.0/src/smartify/state/store.py +370 -0
- smartify_ai-0.1.0/src/smartify/tools/__init__.py +17 -0
- smartify_ai-0.1.0/src/smartify/tools/base.py +196 -0
- smartify_ai-0.1.0/src/smartify/tools/builtin/__init__.py +79 -0
- smartify_ai-0.1.0/src/smartify/tools/builtin/file.py +464 -0
- smartify_ai-0.1.0/src/smartify/tools/builtin/http.py +195 -0
- smartify_ai-0.1.0/src/smartify/tools/builtin/shell.py +137 -0
- smartify_ai-0.1.0/src/smartify/tools/mcp/__init__.py +33 -0
- smartify_ai-0.1.0/src/smartify/tools/mcp/adapter.py +157 -0
- smartify_ai-0.1.0/src/smartify/tools/mcp/client.py +334 -0
- smartify_ai-0.1.0/src/smartify/tools/mcp/registry.py +130 -0
- smartify_ai-0.1.0/src/smartify/validator/__init__.py +0 -0
- smartify_ai-0.1.0/src/smartify/validator/validate.py +271 -0
- smartify_ai-0.1.0/src/smartify/workspace/__init__.py +5 -0
- smartify_ai-0.1.0/src/smartify/workspace/manager.py +248 -0
- smartify_ai-0.1.0/test_e2e.py +73 -0
- smartify_ai-0.1.0/test_e2e_tools.py +67 -0
- smartify_ai-0.1.0/test_parallel_swarm.py +71 -0
- smartify_ai-0.1.0/tests/test_anthropic_adapter.py +141 -0
- smartify_ai-0.1.0/tests/test_api.py +383 -0
- smartify_ai-0.1.0/tests/test_approval.py +832 -0
- smartify_ai-0.1.0/tests/test_breakers.py +249 -0
- smartify_ai-0.1.0/tests/test_checkpoint.py +397 -0
- smartify_ai-0.1.0/tests/test_integration_live.py +447 -0
- smartify_ai-0.1.0/tests/test_mcp.py +299 -0
- smartify_ai-0.1.0/tests/test_openai_adapter.py +350 -0
- smartify_ai-0.1.0/tests/test_orchestrator.py +1074 -0
- smartify_ai-0.1.0/tests/test_resume.py +757 -0
- smartify_ai-0.1.0/tests/test_scheduler.py +191 -0
- smartify_ai-0.1.0/tests/test_spark.py +655 -0
- smartify_ai-0.1.0/tests/test_state.py +219 -0
- smartify_ai-0.1.0/tests/test_tools.py +254 -0
- smartify_ai-0.1.0/tests/test_validator_errors.py +161 -0
- smartify_ai-0.1.0/tests/test_webhook_notifications.py +533 -0
- smartify_ai-0.1.0/tests/test_workspace.py +151 -0
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# Python
|
|
2
|
+
.venv/
|
|
3
|
+
venv/
|
|
4
|
+
env/
|
|
5
|
+
__pycache__/
|
|
6
|
+
*.py[cod]
|
|
7
|
+
*.pyo
|
|
8
|
+
.pytest_cache/
|
|
9
|
+
.mypy_cache/
|
|
10
|
+
*.egg-info/
|
|
11
|
+
.eggs/
|
|
12
|
+
dist/
|
|
13
|
+
build/
|
|
14
|
+
|
|
15
|
+
# Local state and secrets
|
|
16
|
+
*.db
|
|
17
|
+
smartify_state.db
|
|
18
|
+
.env
|
|
19
|
+
.env.*
|
|
20
|
+
!.env.example
|
|
21
|
+
*.local
|
|
22
|
+
|
|
23
|
+
# OS and IDE
|
|
24
|
+
.DS_Store
|
|
25
|
+
.idea/
|
|
26
|
+
.vscode/
|
|
27
|
+
*.swp
|
|
28
|
+
*.swo
|
|
29
|
+
*~
|
|
30
|
+
|
|
31
|
+
# Coverage and tools
|
|
32
|
+
coverage.xml
|
|
33
|
+
htmlcov/
|
|
34
|
+
.ruff_cache/
|
|
@@ -0,0 +1,389 @@
|
|
|
1
|
+
# Smartify Runtime MVP - Design Document
|
|
2
|
+
|
|
3
|
+
**Author:** Umbra
|
|
4
|
+
**Date:** 2026-02-01
|
|
5
|
+
**Status:** Draft
|
|
6
|
+
|
|
7
|
+
## Overview
|
|
8
|
+
|
|
9
|
+
Build a 100% local Python-based MVP runtime for the Smartify Grid specification. This runtime will orchestrate AI agents with guardrails, execute tasks in parallel, and optionally run workloads in an isolated sandbox container.
|
|
10
|
+
|
|
11
|
+
**Goals:**
|
|
12
|
+
- Parse and validate Grid YAML specifications
|
|
13
|
+
- Orchestrate agent execution following the Grid topology
|
|
14
|
+
- Enforce guardrails (token/time/cost limits, breakers)
|
|
15
|
+
- Support parallel task execution
|
|
16
|
+
- API and CLI interface only (no UI)
|
|
17
|
+
|
|
18
|
+
---
|
|
19
|
+
|
|
20
|
+
## Grid Specification Summary
|
|
21
|
+
|
|
22
|
+
### Root Structure
|
|
23
|
+
```yaml
|
|
24
|
+
apiVersion: smartify.ai/v1
|
|
25
|
+
kind: GridSpec
|
|
26
|
+
metadata: # Required - Grid identification
|
|
27
|
+
topology: # Required - Node graph structure
|
|
28
|
+
inputs: # Optional - Input parameters
|
|
29
|
+
environment: # Optional - Variables and secrets
|
|
30
|
+
outputs: # Optional - Artifacts and success criteria
|
|
31
|
+
agents: # Optional - Agent definitions
|
|
32
|
+
tools: # Optional - Tool configuration
|
|
33
|
+
guardrails: # Optional - Limits and breaker actions
|
|
34
|
+
workspace: # Optional - Execution environment
|
|
35
|
+
coordination: # Optional - State management
|
|
36
|
+
observability: # Optional - Logging and events
|
|
37
|
+
ui: # Optional - Dashboard hints
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
### Grid States
|
|
41
|
+
```
|
|
42
|
+
draft → ready → energized → running → completed
|
|
43
|
+
↓
|
|
44
|
+
(paused | stopped | failed)
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
### Node Types
|
|
48
|
+
|
|
49
|
+
**Agent-driven (user-defined):**
|
|
50
|
+
- `controller` - Primary orchestrator (exactly 1 per grid)
|
|
51
|
+
- `relay` - Coordination/delegation node (parent: controller)
|
|
52
|
+
- `substation` - Task execution node (parent: relay)
|
|
53
|
+
|
|
54
|
+
**Runtime-only (not user-defined):**
|
|
55
|
+
- `spark` - Ephemeral helpers spawned automatically by substations
|
|
56
|
+
|
|
57
|
+
**Deterministic nodes:**
|
|
58
|
+
- `foreach` - Fan-out iteration over a list
|
|
59
|
+
- `expr` - Expression evaluation
|
|
60
|
+
- `aggregate` - Fan-in merge of outputs
|
|
61
|
+
- `approval` - Human-in-the-loop checkpoint
|
|
62
|
+
|
|
63
|
+
### Hierarchy
|
|
64
|
+
```
|
|
65
|
+
Grid
|
|
66
|
+
└── Controller (1)
|
|
67
|
+
└── Relay(s) (0-N)
|
|
68
|
+
└── Substation(s) (0-N)
|
|
69
|
+
└── Spark(s) (runtime-spawned, 0-N)
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
### Breaker Types
|
|
73
|
+
| Type | Monitors |
|
|
74
|
+
|------|----------|
|
|
75
|
+
| `tokens` | Input/output token usage |
|
|
76
|
+
| `time` | Runtime duration and task duration |
|
|
77
|
+
| `cost` | API cost per run and per day |
|
|
78
|
+
| `requests` | Requests per minute and concurrent agents |
|
|
79
|
+
|
|
80
|
+
**Breaker Status:** `ok` → `warning` (>80%) → `tripped` (>=100%)
|
|
81
|
+
|
|
82
|
+
**Trip Actions:** `notify`, `pause`, `cooldown`, `require_approval`, `stop`, `downgrade`, `block`
|
|
83
|
+
|
|
84
|
+
### Autonomy Modes
|
|
85
|
+
- `observe` - Read-only, no modifications
|
|
86
|
+
- `recommend` - Suggest changes, require approval
|
|
87
|
+
- `act_with_limits` - Execute within guardrails
|
|
88
|
+
- `full_autonomy` - Execute without restrictions
|
|
89
|
+
|
|
90
|
+
### Trigger Types
|
|
91
|
+
- `manual` - Started by user action
|
|
92
|
+
- `schedule` - Started on a cron schedule
|
|
93
|
+
- `webhook` - Started by HTTP webhook
|
|
94
|
+
- `event` - Started by system event
|
|
95
|
+
|
|
96
|
+
### Assignment Lifecycle
|
|
97
|
+
```
|
|
98
|
+
assigned → accepted → executing → completed
|
|
99
|
+
↓
|
|
100
|
+
blocked → accepted (when unblocked)
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
### Inbox Items
|
|
104
|
+
- `assignment_delivery` - New work to process
|
|
105
|
+
- `signal` - Control messages (pause, resume, stop)
|
|
106
|
+
- `output` - Results from other nodes
|
|
107
|
+
|
|
108
|
+
---
|
|
109
|
+
|
|
110
|
+
## Architecture
|
|
111
|
+
|
|
112
|
+
### Core Components
|
|
113
|
+
|
|
114
|
+
```
|
|
115
|
+
┌─────────────────────────────────────────────────────────────────────┐
|
|
116
|
+
│ Smartify Runtime │
|
|
117
|
+
├─────────────────────────────────────────────────────────────────────┤
|
|
118
|
+
│ │
|
|
119
|
+
│ ┌──────────────┐ ┌──────────────┐ ┌──────────────────────────┐ │
|
|
120
|
+
│ │ CLI │ │ API │ │ Validator │ │
|
|
121
|
+
│ │ Interface │ │ Server │ │ (Grid Spec Checker) │ │
|
|
122
|
+
│ └──────┬───────┘ └──────┬───────┘ └──────────────────────────┘ │
|
|
123
|
+
│ │ │ │
|
|
124
|
+
│ └────────┬────────┘ │
|
|
125
|
+
│ │ │
|
|
126
|
+
│ ┌────────▼────────┐ │
|
|
127
|
+
│ │ Grid Engine │ │
|
|
128
|
+
│ │ (Orchestrator)│ │
|
|
129
|
+
│ └────────┬────────┘ │
|
|
130
|
+
│ │ │
|
|
131
|
+
│ ┌─────────────┼─────────────┐ │
|
|
132
|
+
│ │ │ │ │
|
|
133
|
+
│ ▼ ▼ ▼ │
|
|
134
|
+
│ ┌──────┐ ┌──────────┐ ┌──────────┐ │
|
|
135
|
+
│ │State │ │Scheduler │ │Guardrails│ │
|
|
136
|
+
│ │Store │ │(DAG) │ │Monitor │ │
|
|
137
|
+
│ └──────┘ └────┬─────┘ └──────────┘ │
|
|
138
|
+
│ │ │
|
|
139
|
+
│ ┌───────┴───────┐ │
|
|
140
|
+
│ │ │ │
|
|
141
|
+
│ ▼ ▼ │
|
|
142
|
+
│ ┌──────────┐ ┌──────────────┐ │
|
|
143
|
+
│ │ Agent │ │ Tool │ │
|
|
144
|
+
│ │ Runner │ │ Registry │ │
|
|
145
|
+
│ └────┬─────┘ └──────────────┘ │
|
|
146
|
+
│ │ │
|
|
147
|
+
│ ▼ │
|
|
148
|
+
│ ┌──────────────────────────────────┐ │
|
|
149
|
+
│ │ LLM Provider Adapters │ │
|
|
150
|
+
│ │ (Anthropic, OpenAI, etc.) │ │
|
|
151
|
+
│ └──────────────────────────────────┘ │
|
|
152
|
+
│ │
|
|
153
|
+
└─────────────────────────────────────────────────────────────────────┘
|
|
154
|
+
│
|
|
155
|
+
▼
|
|
156
|
+
┌─────────────────┐
|
|
157
|
+
│ Sandbox │
|
|
158
|
+
│ Container │
|
|
159
|
+
│ (Optional) │
|
|
160
|
+
└─────────────────┘
|
|
161
|
+
```
|
|
162
|
+
|
|
163
|
+
### Module Breakdown
|
|
164
|
+
|
|
165
|
+
1. **CLI Interface** (`smartify/cli/`)
|
|
166
|
+
- `main.py` - Entry point, argument parsing
|
|
167
|
+
- `run.py` - Run a grid from YAML file
|
|
168
|
+
- `validate.py` - Validate grid spec
|
|
169
|
+
- `status.py` - Check running grid status
|
|
170
|
+
- `stop.py` - Stop a running grid
|
|
171
|
+
|
|
172
|
+
2. **API Server** (`smartify/api/`)
|
|
173
|
+
- FastAPI-based REST API
|
|
174
|
+
- Endpoints: `/grids`, `/runs`, `/status`, `/control`
|
|
175
|
+
- WebSocket for real-time updates (optional for MVP)
|
|
176
|
+
|
|
177
|
+
3. **Validator** (`smartify/validator/`)
|
|
178
|
+
- Schema validation (JSON Schema or Pydantic)
|
|
179
|
+
- Semantic validation (topology consistency, no cycles, etc.)
|
|
180
|
+
- Warning/error reporting
|
|
181
|
+
|
|
182
|
+
4. **Grid Engine** (`smartify/engine/`)
|
|
183
|
+
- `grid.py` - Grid state machine
|
|
184
|
+
- `orchestrator.py` - Main execution loop
|
|
185
|
+
- `scheduler.py` - DAG-based task scheduling
|
|
186
|
+
- `executor.py` - Node execution dispatch
|
|
187
|
+
|
|
188
|
+
5. **State Store** (`smartify/state/`)
|
|
189
|
+
- In-memory store (MVP)
|
|
190
|
+
- SQLite persistence (optional)
|
|
191
|
+
- Grid state, node states, assignment tracking
|
|
192
|
+
|
|
193
|
+
6. **Agent Runner** (`smartify/agents/`)
|
|
194
|
+
- `runner.py` - Agent execution wrapper
|
|
195
|
+
- `adapters/` - LLM provider adapters (Anthropic, OpenAI)
|
|
196
|
+
- `prompts/` - Built-in system prompts
|
|
197
|
+
|
|
198
|
+
7. **Guardrails** (`smartify/guardrails/`)
|
|
199
|
+
- `breakers.py` - Breaker implementation
|
|
200
|
+
- `monitor.py` - Token/time/cost tracking
|
|
201
|
+
- `actions.py` - Trip action handlers
|
|
202
|
+
|
|
203
|
+
8. **Tools** (`smartify/tools/`)
|
|
204
|
+
- `registry.py` - Tool registration
|
|
205
|
+
- `builtin/` - Built-in tools (terminal, filesystem, git, browser)
|
|
206
|
+
|
|
207
|
+
9. **Workspace** (`smartify/workspace/`)
|
|
208
|
+
- `local.py` - Local filesystem workspace
|
|
209
|
+
- `container.py` - Docker container workspace
|
|
210
|
+
- `sandbox.py` - Sandbox integration
|
|
211
|
+
|
|
212
|
+
---
|
|
213
|
+
|
|
214
|
+
## Data Models (Pydantic)
|
|
215
|
+
|
|
216
|
+
```python
|
|
217
|
+
# Core models
|
|
218
|
+
class GridSpec(BaseModel):
|
|
219
|
+
apiVersion: str = "smartify.ai/v1"
|
|
220
|
+
kind: str = "GridSpec"
|
|
221
|
+
metadata: MetadataSpec
|
|
222
|
+
topology: TopologySpec
|
|
223
|
+
inputs: Optional[List[InputSpec]]
|
|
224
|
+
environment: Optional[EnvironmentSpec]
|
|
225
|
+
outputs: Optional[OutputsSpec]
|
|
226
|
+
agents: Optional[Dict[str, AgentSpec]]
|
|
227
|
+
tools: Optional[ToolsSpec]
|
|
228
|
+
guardrails: Optional[GuardrailsSpec]
|
|
229
|
+
workspace: Optional[WorkspaceSpec]
|
|
230
|
+
|
|
231
|
+
class NodeSpec(BaseModel):
|
|
232
|
+
id: str
|
|
233
|
+
kind: NodeKind # controller, relay, substation, foreach, expr, aggregate, approval
|
|
234
|
+
name: str
|
|
235
|
+
description: Optional[str]
|
|
236
|
+
parent: Optional[str]
|
|
237
|
+
agent: Optional[str]
|
|
238
|
+
capabilities: List[str] = []
|
|
239
|
+
when: Optional[str] # Expression
|
|
240
|
+
parallel: bool = True
|
|
241
|
+
runAfter: Optional[List[str]]
|
|
242
|
+
retry: Optional[RetrySpec]
|
|
243
|
+
outputSchema: Optional[dict]
|
|
244
|
+
|
|
245
|
+
class BreakerSpec(BaseModel):
|
|
246
|
+
tokens: Optional[TokenLimits]
|
|
247
|
+
time: Optional[TimeLimits]
|
|
248
|
+
cost: Optional[CostLimits]
|
|
249
|
+
requests: Optional[RequestLimits]
|
|
250
|
+
|
|
251
|
+
class BreakerActions(BaseModel):
|
|
252
|
+
onTokensLimit: TripAction = TripAction.PAUSE
|
|
253
|
+
onTimeLimit: TripAction = TripAction.STOP
|
|
254
|
+
onCostLimit: TripAction = TripAction.PAUSE
|
|
255
|
+
onRequestsLimit: TripAction = TripAction.COOLDOWN
|
|
256
|
+
```
|
|
257
|
+
|
|
258
|
+
---
|
|
259
|
+
|
|
260
|
+
## MVP Scope
|
|
261
|
+
|
|
262
|
+
### Phase 1: Core Infrastructure (Week 1)
|
|
263
|
+
- [ ] Grid YAML parsing and validation
|
|
264
|
+
- [ ] Pydantic models for all spec types
|
|
265
|
+
- [ ] CLI skeleton (validate, run)
|
|
266
|
+
- [ ] In-memory state store
|
|
267
|
+
|
|
268
|
+
### Phase 2: Execution Engine (Week 1-2)
|
|
269
|
+
- [ ] DAG-based scheduler
|
|
270
|
+
- [ ] Node execution dispatch
|
|
271
|
+
- [ ] Assignment delivery system
|
|
272
|
+
- [ ] Parent-child execution flow
|
|
273
|
+
|
|
274
|
+
### Phase 3: Agent Integration (Week 2)
|
|
275
|
+
- [ ] LLM provider adapters (start with Anthropic)
|
|
276
|
+
- [ ] Built-in system prompts
|
|
277
|
+
- [ ] Tool execution framework
|
|
278
|
+
- [ ] Basic tools (terminal, filesystem)
|
|
279
|
+
|
|
280
|
+
### Phase 4: Guardrails (Week 2-3)
|
|
281
|
+
- [ ] Token counting and limits
|
|
282
|
+
- [ ] Time tracking
|
|
283
|
+
- [ ] Breaker implementation
|
|
284
|
+
- [ ] Trip actions (pause, stop, notify)
|
|
285
|
+
|
|
286
|
+
### Phase 5: Polish & Testing (Week 3-4)
|
|
287
|
+
- [ ] API server
|
|
288
|
+
- [ ] Comprehensive validation
|
|
289
|
+
- [ ] Error handling
|
|
290
|
+
- [ ] Test suite
|
|
291
|
+
- [ ] Documentation
|
|
292
|
+
|
|
293
|
+
---
|
|
294
|
+
|
|
295
|
+
## CLI Commands
|
|
296
|
+
|
|
297
|
+
```bash
|
|
298
|
+
# Validate a grid spec
|
|
299
|
+
smartify validate grid.yaml
|
|
300
|
+
|
|
301
|
+
# Run a grid
|
|
302
|
+
smartify run grid.yaml --input project_name=myapp
|
|
303
|
+
|
|
304
|
+
# Check status of running grid
|
|
305
|
+
smartify status <run_id>
|
|
306
|
+
|
|
307
|
+
# Stop a running grid
|
|
308
|
+
smartify stop <run_id>
|
|
309
|
+
|
|
310
|
+
# List recent runs
|
|
311
|
+
smartify list
|
|
312
|
+
|
|
313
|
+
# Show grid info
|
|
314
|
+
smartify info grid.yaml
|
|
315
|
+
```
|
|
316
|
+
|
|
317
|
+
---
|
|
318
|
+
|
|
319
|
+
## API Endpoints
|
|
320
|
+
|
|
321
|
+
```
|
|
322
|
+
POST /api/v1/grids # Create/upload grid spec
|
|
323
|
+
GET /api/v1/grids # List grids
|
|
324
|
+
GET /api/v1/grids/{id} # Get grid details
|
|
325
|
+
DELETE /api/v1/grids/{id} # Delete grid
|
|
326
|
+
|
|
327
|
+
POST /api/v1/runs # Start a grid run
|
|
328
|
+
GET /api/v1/runs # List runs
|
|
329
|
+
GET /api/v1/runs/{id} # Get run status
|
|
330
|
+
POST /api/v1/runs/{id}/control # pause, resume, stop
|
|
331
|
+
GET /api/v1/runs/{id}/nodes # List node states
|
|
332
|
+
GET /api/v1/runs/{id}/logs # Get execution logs
|
|
333
|
+
|
|
334
|
+
POST /api/v1/validate # Validate a grid spec
|
|
335
|
+
```
|
|
336
|
+
|
|
337
|
+
---
|
|
338
|
+
|
|
339
|
+
## Sandbox Integration
|
|
340
|
+
|
|
341
|
+
The runtime can optionally execute agent tasks in an isolated Docker container (similar to the sandbox we built). Configuration via `workspace` section:
|
|
342
|
+
|
|
343
|
+
```yaml
|
|
344
|
+
workspace:
|
|
345
|
+
type: container
|
|
346
|
+
config:
|
|
347
|
+
image: smartify-sandbox:latest
|
|
348
|
+
resources:
|
|
349
|
+
cpus: 2
|
|
350
|
+
memoryMb: 4096
|
|
351
|
+
mounts:
|
|
352
|
+
- src: ./output
|
|
353
|
+
dst: /workspace/output
|
|
354
|
+
```
|
|
355
|
+
|
|
356
|
+
---
|
|
357
|
+
|
|
358
|
+
## Key Design Decisions
|
|
359
|
+
|
|
360
|
+
1. **Python-based** - Easier integration with ML/AI ecosystem, fast development
|
|
361
|
+
2. **Pydantic for validation** - Strong typing, automatic validation, serialization
|
|
362
|
+
3. **DAG scheduler** - Natural fit for topology with dependencies
|
|
363
|
+
4. **In-memory first** - Fast iteration, add persistence later
|
|
364
|
+
5. **Adapter pattern for LLMs** - Easy to add new providers
|
|
365
|
+
6. **Built-in tools** - Start minimal, expand based on need
|
|
366
|
+
|
|
367
|
+
---
|
|
368
|
+
|
|
369
|
+
## Risks & Mitigations
|
|
370
|
+
|
|
371
|
+
| Risk | Mitigation |
|
|
372
|
+
|------|------------|
|
|
373
|
+
| Scope creep | Strict MVP scope, defer features to backlog |
|
|
374
|
+
| LLM rate limits | Implement proper backoff, cost tracking |
|
|
375
|
+
| Container overhead | Make sandbox optional, default to local |
|
|
376
|
+
| Complex topologies | Start with simple parent-child, add edges later |
|
|
377
|
+
|
|
378
|
+
---
|
|
379
|
+
|
|
380
|
+
## Next Steps
|
|
381
|
+
|
|
382
|
+
1. Set up Python project structure
|
|
383
|
+
2. Implement Pydantic models for Grid spec
|
|
384
|
+
3. Build validator CLI command
|
|
385
|
+
4. Implement basic orchestrator
|
|
386
|
+
5. Add Anthropic adapter
|
|
387
|
+
6. Implement breakers
|
|
388
|
+
7. Build API server
|
|
389
|
+
8. Test with example grids
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
# Smartify Runtime - Implementation Status
|
|
2
|
+
|
|
3
|
+
## Current Status
|
|
4
|
+
- **Tests:** 170/170 passing ✅
|
|
5
|
+
- **LOC:** ~8,500 lines Python
|
|
6
|
+
- **Core:** Orchestrator, Scheduler, Validator, API Server, Anthropic + OpenAI Adapters, Breakers, Tool Registry + Builtins, State Store, Webhook Notifications, Checkpoint/Resume
|
|
7
|
+
|
|
8
|
+
## Completed ✅
|
|
9
|
+
|
|
10
|
+
### 1. Builtin Tools (`tools/`)
|
|
11
|
+
- `ToolRegistry` - Registration, lookup, format conversion
|
|
12
|
+
- `ShellTool` - Command execution with timeout, security checks
|
|
13
|
+
- `FileReadTool` / `FileWriteTool` / `FileListTool` / `FileDeleteTool`
|
|
14
|
+
- `HttpTool` - HTTP requests with host filtering
|
|
15
|
+
- Integrated into orchestrator
|
|
16
|
+
|
|
17
|
+
### 2. State Persistence (`state/`)
|
|
18
|
+
- `RunRecord` / `NodeOutput` data classes
|
|
19
|
+
- `InMemoryStore` - For testing
|
|
20
|
+
- `SQLiteStore` - Persistent storage with full CRUD
|
|
21
|
+
- Run history, status tracking, cost/token aggregation
|
|
22
|
+
|
|
23
|
+
### 3. Validator Improvements
|
|
24
|
+
- Pre-validation for required fields (apiVersion, metadata.id, etc.)
|
|
25
|
+
- 11 error detection tests passing
|
|
26
|
+
- Catches: missing fields, duplicate IDs, invalid parents, cycles
|
|
27
|
+
|
|
28
|
+
### 4. CLI Commands
|
|
29
|
+
- `validate` - Grid validation ✅
|
|
30
|
+
- `run` - Execute grids ✅
|
|
31
|
+
- `status` - Check run status ✅
|
|
32
|
+
- `stop` - Stop running grids ✅
|
|
33
|
+
- `list` - List runs with filters ✅
|
|
34
|
+
- `serve` - API server ✅
|
|
35
|
+
- `info` - Grid inspection ✅
|
|
36
|
+
|
|
37
|
+
### 5. Example Grids
|
|
38
|
+
- `minimal.yaml` - Basic single-controller grid
|
|
39
|
+
- `file-creator.yaml` - Tool execution test
|
|
40
|
+
- `parallel-swarm.yaml` - Multi-agent parallel execution with guardrails
|
|
41
|
+
|
|
42
|
+
## Remaining Gaps
|
|
43
|
+
|
|
44
|
+
### Medium Priority
|
|
45
|
+
- **Workspace isolation** - Sandboxed file operations per run
|
|
46
|
+
- **Spark nodes** - Dynamic node spawning
|
|
47
|
+
- **Approval flow** - Notifications, webhook callbacks
|
|
48
|
+
|
|
49
|
+
### Lower Priority
|
|
50
|
+
- **Retry improvements** - Exponential backoff, configurable strategies
|
|
51
|
+
- **Streaming output** - Real-time node progress
|
|
52
|
+
- **Metrics export** - Prometheus/OpenTelemetry integration
|
|
53
|
+
|
|
54
|
+
## E2E Verified
|
|
55
|
+
- Grid loading and validation
|
|
56
|
+
- Orchestrator lifecycle (draft → energized → running → completed)
|
|
57
|
+
- Tool execution (file_write creates files)
|
|
58
|
+
- Parallel node execution
|
|
59
|
+
- Token/cost tracking
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
# Smartify Runtime — Launch Plan
|
|
2
|
+
|
|
3
|
+
Plan for open-source launch: MCP integration, repo cleanup, and license/docs alignment.
|
|
4
|
+
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
## 1. MCP (Model Context Protocol) Integration
|
|
8
|
+
|
|
9
|
+
**Goal:** Allow grids to use tools from external MCP servers so existing MCP integrations work plug-and-play with Smartify agent swarms.
|
|
10
|
+
|
|
11
|
+
**Reference:** [MCP Python SDK](https://github.com/modelcontextprotocol/python-sdk) — official client; supports stdio, SSE, Streamable HTTP. Install: `pip install mcp` (Python ≥3.10).
|
|
12
|
+
|
|
13
|
+
### 1.1 Architecture
|
|
14
|
+
|
|
15
|
+
- **MCP as tool source:** One or more MCP servers are configured per run (or globally). The runtime connects as an MCP **client**, calls `tools/list` to discover tools, and exposes each MCP tool as a Smartify `Tool` so the existing `ToolRegistry` and orchestrator can use them.
|
|
16
|
+
- **Adapter pattern:** Add `McpToolAdapter` (or `McpServerBridge`) that:
|
|
17
|
+
- Connects to an MCP server (stdio command, SSE URL, or Streamable HTTP URL).
|
|
18
|
+
- Lists tools via MCP `tools/list`.
|
|
19
|
+
- For each MCP tool, creates a wrapper that implements Smartify's `Tool` interface (`name`, `description`, `parameters` from MCP schema, `execute` → MCP `tools/call`).
|
|
20
|
+
- **Naming:** MCP tools can clash with builtins. Use a prefix per server (e.g. `mcp_fileread` for server label `mcp`) or a configurable prefix in grid/context so nodes can refer to `mcp_fileread` or `filesystem.read_file`.
|
|
21
|
+
|
|
22
|
+
### 1.2 Phases
|
|
23
|
+
|
|
24
|
+
#### Phase A: Dependency and MCP client wrapper (foundation)
|
|
25
|
+
|
|
26
|
+
- [x] Add optional dependency: `mcp` in `pyproject.toml` (e.g. `mcp>=1.0` in `[project.optional-dependencies]` as `mcp` or under `integrations`).
|
|
27
|
+
- [x] Create `src/smartify/tools/mcp/` package:
|
|
28
|
+
- [x] `__init__.py` — export public API.
|
|
29
|
+
- [x] `client.py` — thin wrapper around MCP Python SDK client:
|
|
30
|
+
- Connect (stdio command list, or SSE/HTTP URL) from config.
|
|
31
|
+
- `list_tools() -> List[McpToolDef]` (name, description, parameters schema).
|
|
32
|
+
- `call_tool(name: str, arguments: dict) -> ToolResult` (async).
|
|
33
|
+
- [x] Handle connection lifecycle (connect when building registry or on first use; disconnect on shutdown or run end). Prefer one client per MCP server process/URL.
|
|
34
|
+
|
|
35
|
+
#### Phase B: Smartify `Tool` adapter for MCP
|
|
36
|
+
|
|
37
|
+
- [x] In `src/smartify/tools/mcp/`, add `adapter.py`:
|
|
38
|
+
- [x] Class `McpToolWrapper(Tool)`:
|
|
39
|
+
- Takes an MCP client (or server session) + one MCP tool definition.
|
|
40
|
+
- `name`: MCP tool name, optionally prefixed (e.g. `mcp_filesystem_read`) to avoid clashes.
|
|
41
|
+
- `description`, `parameters`: from MCP tool definition (map to our JSON Schema shape if needed).
|
|
42
|
+
- `execute(**kwargs)`: call MCP `tools/call` with the tool name and kwargs; map MCP result (e.g. content list or error) to `ToolResult(success=..., output=..., error=...)`.
|
|
43
|
+
- [x] Add tests (unit: mock MCP client; optional integration: real MCP server in CI or behind `pytest -m mcp`).
|
|
44
|
+
|
|
45
|
+
#### Phase C: Registry integration
|
|
46
|
+
|
|
47
|
+
- [x] Add `McpRegistryBuilder` or a function `register_mcp_server(registry: ToolRegistry, config: McpServerConfig) -> None`:
|
|
48
|
+
- Connect to MCP server, list tools, create `McpToolWrapper` per tool, `registry.register(wrapper)` for each.
|
|
49
|
+
- Config: transport type (stdio / sse / streamable_http), command or URL, optional tool name prefix, optional allow-list of tool names.
|
|
50
|
+
- [x] Ensure `create_builtin_registry()` (or the factory used by the orchestrator) can accept optional MCP configs and call `register_mcp_server` for each, so builtins + MCP tools are in one registry.
|
|
51
|
+
|
|
52
|
+
#### Phase D: Grid YAML and runtime wiring
|
|
53
|
+
|
|
54
|
+
- [x] Extend grid spec (e.g. in `models/grid.py`):
|
|
55
|
+
- [x] Add `McpServerSpec` (e.g. `id`, `transport: stdio | sse | streamable_http`, `command` or `url`, optional `prefix`, optional `tools: [names]`).
|
|
56
|
+
- [x] In `ToolsSpec`, add `mcpServers: List[McpServerSpec]` (or under `tools.mcp`).
|
|
57
|
+
- [x] Validator: validate `mcpServers` entries (required fields, valid transport).
|
|
58
|
+
- [x] Orchestrator (or the place that builds `ToolRegistry` for a run):
|
|
59
|
+
- [x] When building the registry for a run, read grid's `tools.mcpServers` (or equivalent), and for each server call `register_mcp_server(registry, server_config)`.
|
|
60
|
+
- [x] If MCP is optional dependency, catch import error and raise a clear "install smartify[mcp] to use MCP servers" (or similar).
|
|
61
|
+
- [x] Document in README: "Using MCP servers" with a small example (one stdio server and one tool used in a grid).
|
|
62
|
+
|
|
63
|
+
#### Phase E: Docs and optional extras
|
|
64
|
+
|
|
65
|
+
- [x] README section: "Tool integrations" — builtins + MCP; link to MCP spec/sdk.
|
|
66
|
+
- [x] Example grid YAML that uses one MCP server (e.g. filesystem or a simple test server) so users can run it with `smartify run examples/grid-with-mcp.yaml`.
|
|
67
|
+
- [x] Optional: `smartify[mcp]` or `smartify[integrations]` extra in pyproject so MCP is opt-in for minimal installs.
|
|
68
|
+
|
|
69
|
+
### 1.3 Out of scope for initial launch (backlog)
|
|
70
|
+
|
|
71
|
+
- MCP resources/prompts (only tools first).
|
|
72
|
+
- Dynamic tool approval for MCP calls (can add later if needed).
|
|
73
|
+
- Caching or connection pooling across runs (keep simple: connect per run or per process).
|
|
74
|
+
|
|
75
|
+
---
|
|
76
|
+
|
|
77
|
+
## 2. Repo cleanup (pre-push)
|
|
78
|
+
|
|
79
|
+
### 2.1 Add a proper `.gitignore`
|
|
80
|
+
|
|
81
|
+
- [x] Add `.gitignore` at repo root with:
|
|
82
|
+
- Python: `.venv/`, `venv/`, `env/`, `__pycache__/`, `*.py[cod]`, `*.pyo`, `.pytest_cache/`, `.mypy_cache/`, `*.egg-info/`, `dist/`, `build/`
|
|
83
|
+
- Local data: `*.db`, `smartify_state.db`, `.env`, `*.local`
|
|
84
|
+
- OS/IDE: `.DS_Store`, `.idea/`, `*.swp`, `*.swo`
|
|
85
|
+
- Optional: `coverage.xml`, `htmlcov/` if you use coverage
|
|
86
|
+
|
|
87
|
+
### 2.2 Files to delete before pushing to Git
|
|
88
|
+
|
|
89
|
+
- [x] **`ai_healthcare_research.md`** — domain-specific research; not part of runtime.
|
|
90
|
+
- [x] **`AI_Healthcare_Historical_Research.md`** — same.
|
|
91
|
+
- [x] **`smartify_state.db`** — runtime SQLite DB; should not be committed (and is ignored via `.gitignore`).
|
|
92
|
+
- [x] **`.DS_Store`** — macOS metadata (and ignored).
|
|
93
|
+
- **Root-level E2E scripts — keep them:**
|
|
94
|
+
**`test_e2e.py`**, **`test_e2e_tools.py`**, **`test_parallel_swarm.py`** are manual smoke tests that run the **actual example YAML files** (`examples/minimal.yaml`, `file-creator.yaml`, `parallel-swarm.yaml`) with a real API. They complement `tests/test_integration_live.py` (which uses inline YAML and pytest). Use them for quick checks after changing examples: `python test_e2e.py`, `python test_e2e_tools.py`, `python test_parallel_swarm.py "topic"`.
|
|
95
|
+
|
|
96
|
+
### 2.3 Optional
|
|
97
|
+
|
|
98
|
+
- Move or trim **`GAPS.md`** (e.g. move to `docs/` or fold into README "Roadmap") so the repo reads as launch-ready.
|
|
99
|
+
|
|
100
|
+
---
|
|
101
|
+
|
|
102
|
+
## 3. License and project alignment
|
|
103
|
+
|
|
104
|
+
### 3.1 Add LICENSE file
|
|
105
|
+
|
|
106
|
+
- [x] Add **`LICENSE`** at repo root.
|
|
107
|
+
- [x] Use **MIT** to match `pyproject.toml` (already "MIT" and "License :: OSI Approved :: MIT License").
|
|
108
|
+
- [x] In LICENSE, set **Copyright (c) YEAR Smartify** (or your legal entity) and keep standard MIT text.
|
|
109
|
+
|
|
110
|
+
### 3.2 Align README and pyproject.toml
|
|
111
|
+
|
|
112
|
+
- [x] **README.md:** Change "License" section from "BSL 1.1" to **"MIT"** and add a one-line pointer: "See [LICENSE](LICENSE)."
|
|
113
|
+
- [x] **pyproject.toml:** Leave as-is (already MIT); no change unless you later switch to BSL 1.1 (then update both LICENSE file and both docs).
|
|
114
|
+
|
|
115
|
+
---
|
|
116
|
+
|
|
117
|
+
## Checklist summary
|
|
118
|
+
|
|
119
|
+
| Item | Status |
|
|
120
|
+
|------|--------|
|
|
121
|
+
| **LAUNCH_PLAN.md** (this file) | ✅ Done |
|
|
122
|
+
| **.gitignore** | ✅ Done |
|
|
123
|
+
| Delete healthcare research files | ✅ Done |
|
|
124
|
+
| Delete `smartify_state.db` / .DS_Store | ✅ Done |
|
|
125
|
+
| Keep root-level E2E scripts (restored) | ✅ Done |
|
|
126
|
+
| **LICENSE** (MIT) | ✅ Done |
|
|
127
|
+
| README license → MIT | ✅ Done |
|
|
128
|
+
| **MCP integration** | ✅ Done |
|
|
129
|
+
|
|
130
|
+
---
|
|
131
|
+
|
|
132
|
+
## MCP Integration Complete
|
|
133
|
+
|
|
134
|
+
All phases implemented:
|
|
135
|
+
|
|
136
|
+
- **Phase A**: Added optional `mcp` dependency in pyproject.toml; created `src/smartify/tools/mcp/` package with `client.py` (MCP client wrapper supporting stdio/sse/streamable_http).
|
|
137
|
+
- **Phase B**: Created `adapter.py` with `McpToolWrapper(Tool)` that wraps MCP tools for use in Smartify.
|
|
138
|
+
- **Phase C**: Created `registry.py` with `register_mcp_server()` function to register MCP server tools into a ToolRegistry.
|
|
139
|
+
- **Phase D**: Added `McpServerSpec` to `models/grid.py`, added `mcpServers` to `ToolsSpec`, wired orchestrator to auto-register MCP servers on `energize()`.
|
|
140
|
+
- **Phase E**: Updated README with MCP section, added `examples/mcp-filesystem.yaml`, added `tests/test_mcp.py` (14 tests passing).
|
|
141
|
+
|
|
142
|
+
**Tests**: 355 passing (excluding OpenAI adapter tests with pre-existing httpx version conflict from mcp package).
|
|
143
|
+
|
|
144
|
+
The repo is now ready for open-source launch with "plug and play with existing tool integrations and MCP" and multi-agent providers.
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 Smartify
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|