taskmux 0.2.4__tar.gz → 0.2.6__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {taskmux-0.2.4 → taskmux-0.2.6}/PKG-INFO +107 -9
- {taskmux-0.2.4 → taskmux-0.2.6}/README.md +106 -8
- {taskmux-0.2.4 → taskmux-0.2.6}/pyproject.toml +1 -1
- {taskmux-0.2.4 → taskmux-0.2.6}/taskmux/config.py +8 -0
- {taskmux-0.2.4 → taskmux-0.2.6}/taskmux/daemon.py +59 -2
- {taskmux-0.2.4 → taskmux-0.2.6}/taskmux/models.py +4 -0
- {taskmux-0.2.4 → taskmux-0.2.6}/taskmux/tmux_manager.py +204 -14
- {taskmux-0.2.4 → taskmux-0.2.6}/.gitignore +0 -0
- {taskmux-0.2.4 → taskmux-0.2.6}/LICENSE +0 -0
- {taskmux-0.2.4 → taskmux-0.2.6}/taskmux/__init__.py +0 -0
- {taskmux-0.2.4 → taskmux-0.2.6}/taskmux/agent.py +0 -0
- {taskmux-0.2.4 → taskmux-0.2.6}/taskmux/cli.py +0 -0
- {taskmux-0.2.4 → taskmux-0.2.6}/taskmux/hooks.py +0 -0
- {taskmux-0.2.4 → taskmux-0.2.6}/taskmux/init.py +0 -0
- {taskmux-0.2.4 → taskmux-0.2.6}/taskmux/main.py +0 -0
- {taskmux-0.2.4 → taskmux-0.2.6}/taskmux/templates/claude.md +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: taskmux
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.6
|
|
4
4
|
Summary: Modern tmux-based task manager for LLM development tools
|
|
5
5
|
Project-URL: Homepage, https://github.com/nc9/taskmux
|
|
6
6
|
Project-URL: Repository, https://github.com/nc9/taskmux
|
|
@@ -104,6 +104,67 @@ command = "docker compose up postgres"
|
|
|
104
104
|
auto_start = false
|
|
105
105
|
```
|
|
106
106
|
|
|
107
|
+
## Full Example
|
|
108
|
+
|
|
109
|
+
A full-stack app with a database, API server, and frontend — using health checks to ensure each service is ready before starting its dependents:
|
|
110
|
+
|
|
111
|
+
```toml
|
|
112
|
+
name = "fullstack-app"
|
|
113
|
+
|
|
114
|
+
[tasks.db]
|
|
115
|
+
command = "docker compose up postgres redis"
|
|
116
|
+
health_check = "pg_isready -h localhost -p 5432"
|
|
117
|
+
health_interval = 3
|
|
118
|
+
|
|
119
|
+
[tasks.migrate]
|
|
120
|
+
command = "python manage.py migrate && echo 'done' && sleep infinity"
|
|
121
|
+
cwd = "apps/api"
|
|
122
|
+
depends_on = ["db"]
|
|
123
|
+
health_check = "test -f .migrate-complete"
|
|
124
|
+
|
|
125
|
+
[tasks.api]
|
|
126
|
+
command = "python manage.py runserver 0.0.0.0:8000"
|
|
127
|
+
cwd = "apps/api"
|
|
128
|
+
depends_on = ["migrate"]
|
|
129
|
+
health_check = "curl -sf http://localhost:8000/health"
|
|
130
|
+
|
|
131
|
+
[tasks.worker]
|
|
132
|
+
command = "celery -A myapp worker -l info"
|
|
133
|
+
cwd = "apps/api"
|
|
134
|
+
depends_on = ["db"]
|
|
135
|
+
|
|
136
|
+
[tasks.web]
|
|
137
|
+
command = "bun dev"
|
|
138
|
+
cwd = "apps/web"
|
|
139
|
+
depends_on = ["api"]
|
|
140
|
+
health_check = "curl -sf http://localhost:3000"
|
|
141
|
+
|
|
142
|
+
[tasks.storybook]
|
|
143
|
+
command = "bun storybook"
|
|
144
|
+
cwd = "apps/web"
|
|
145
|
+
auto_start = false
|
|
146
|
+
```
|
|
147
|
+
|
|
148
|
+
What happens on `taskmux start`:
|
|
149
|
+
|
|
150
|
+
1. **db** starts first (no dependencies)
|
|
151
|
+
2. **migrate** and **worker** wait for db's health check (`pg_isready`) to pass
|
|
152
|
+
3. **api** waits for migrate's health check
|
|
153
|
+
4. **web** waits for api's health check (`curl localhost:8000/health`)
|
|
154
|
+
5. **storybook** is skipped (`auto_start = false`) — start it manually with `taskmux start storybook`
|
|
155
|
+
|
|
156
|
+
```bash
|
|
157
|
+
taskmux start # Starts everything in dependency order
|
|
158
|
+
taskmux logs # Interleaved logs from all tasks
|
|
159
|
+
taskmux logs -g "ERROR" # Grep all tasks for errors
|
|
160
|
+
taskmux logs api # Logs from just the API
|
|
161
|
+
taskmux logs -f api # Follow API logs live
|
|
162
|
+
taskmux health # Health check table
|
|
163
|
+
taskmux inspect api # JSON state for a single task
|
|
164
|
+
taskmux restart worker # Restart just the worker
|
|
165
|
+
taskmux start storybook # Start a manual task
|
|
166
|
+
```
|
|
167
|
+
|
|
107
168
|
## Commands
|
|
108
169
|
|
|
109
170
|
```bash
|
|
@@ -124,10 +185,13 @@ taskmux remove <task> # Remove task from config
|
|
|
124
185
|
taskmux inspect <task> # JSON task state (pid, command, health)
|
|
125
186
|
|
|
126
187
|
# Logs
|
|
127
|
-
taskmux logs
|
|
128
|
-
taskmux logs
|
|
129
|
-
taskmux logs -
|
|
130
|
-
taskmux logs <task>
|
|
188
|
+
taskmux logs # Interleaved logs from all tasks
|
|
189
|
+
taskmux logs <task> # Show recent logs for a task
|
|
190
|
+
taskmux logs -f # Attach to session (switch windows with tmux keybinds)
|
|
191
|
+
taskmux logs -f <task> # Follow a task's logs live
|
|
192
|
+
taskmux logs -n 200 <task> # Last N lines
|
|
193
|
+
taskmux logs -g "error" # Search all tasks
|
|
194
|
+
taskmux logs <task> -g "error" # Search one task
|
|
131
195
|
taskmux logs <task> -g "error" -C 5 # Grep with context lines
|
|
132
196
|
|
|
133
197
|
# Init
|
|
@@ -161,12 +225,20 @@ after_stop = "echo done"
|
|
|
161
225
|
|
|
162
226
|
[tasks.server]
|
|
163
227
|
command = "python manage.py runserver"
|
|
228
|
+
cwd = "apps/api"
|
|
229
|
+
health_check = "curl -sf http://localhost:8000/health"
|
|
230
|
+
depends_on = ["db"]
|
|
164
231
|
|
|
165
232
|
[tasks.server.hooks]
|
|
166
233
|
before_start = "python manage.py migrate"
|
|
167
234
|
|
|
235
|
+
[tasks.db]
|
|
236
|
+
command = "docker compose up postgres"
|
|
237
|
+
health_check = "pg_isready -h localhost"
|
|
238
|
+
|
|
168
239
|
[tasks.worker]
|
|
169
240
|
command = "celery worker -A myapp"
|
|
241
|
+
depends_on = ["db"]
|
|
170
242
|
|
|
171
243
|
[tasks.tailwind]
|
|
172
244
|
command = "npx tailwindcss -w"
|
|
@@ -185,8 +257,31 @@ auto_start = false
|
|
|
185
257
|
| `hooks.after_stop` | — | Run after stopping tasks |
|
|
186
258
|
| `tasks.<name>.command` | — | Shell command to run |
|
|
187
259
|
| `tasks.<name>.auto_start` | `true` | Start with `taskmux start` |
|
|
260
|
+
| `tasks.<name>.cwd` | — | Working directory for the task |
|
|
261
|
+
| `tasks.<name>.health_check` | — | Shell command to check health (exit 0 = healthy) |
|
|
262
|
+
| `tasks.<name>.health_interval` | `10` | Seconds between health checks |
|
|
263
|
+
| `tasks.<name>.health_timeout` | `5` | Seconds before health check times out |
|
|
264
|
+
| `tasks.<name>.health_retries` | `3` | Consecutive failures before "unhealthy" |
|
|
265
|
+
| `tasks.<name>.depends_on` | `[]` | Task names that must be healthy before this task starts |
|
|
188
266
|
| `tasks.<name>.hooks.*` | — | Per-task lifecycle hooks (same fields as global) |
|
|
189
267
|
|
|
268
|
+
### Dependency Ordering
|
|
269
|
+
|
|
270
|
+
Tasks with `depends_on` are started in topological order. Before starting a task, taskmux waits for each dependency's health check to pass (up to `health_retries * health_interval` seconds). If a dependency never becomes healthy, the dependent task is skipped with a warning.
|
|
271
|
+
|
|
272
|
+
Circular dependencies and references to nonexistent tasks are rejected at config load time.
|
|
273
|
+
|
|
274
|
+
When starting a single task with `taskmux start <task>`, dependencies are not auto-started — you get a warning if they aren't running.
|
|
275
|
+
|
|
276
|
+
### Health Checks
|
|
277
|
+
|
|
278
|
+
If `health_check` is set, taskmux runs it as a shell command. Exit code 0 means healthy. If not set, taskmux falls back to checking if the tmux pane has a running process (not just a shell prompt).
|
|
279
|
+
|
|
280
|
+
Health checks are used by:
|
|
281
|
+
- `taskmux health` — shows a table of all task health
|
|
282
|
+
- `taskmux start` — waits for dependencies to be healthy before starting dependents
|
|
283
|
+
- `taskmux daemon` — continuously monitors and auto-restarts unhealthy tasks
|
|
284
|
+
|
|
190
285
|
### Hook Cascade
|
|
191
286
|
|
|
192
287
|
Hooks fire in this order:
|
|
@@ -212,14 +307,17 @@ Use `--defaults` to skip prompts (CI/automation).
|
|
|
212
307
|
|
|
213
308
|
```json
|
|
214
309
|
{
|
|
215
|
-
"name": "
|
|
216
|
-
"command": "
|
|
310
|
+
"name": "api",
|
|
311
|
+
"command": "python manage.py runserver 0.0.0.0:8000",
|
|
217
312
|
"auto_start": true,
|
|
313
|
+
"cwd": "apps/api",
|
|
314
|
+
"health_check": "curl -sf http://localhost:8000/health",
|
|
315
|
+
"depends_on": ["db"],
|
|
218
316
|
"running": true,
|
|
219
317
|
"healthy": true,
|
|
220
318
|
"pid": "12345",
|
|
221
|
-
"pane_current_command": "
|
|
222
|
-
"pane_current_path": "/home/user/project",
|
|
319
|
+
"pane_current_command": "python",
|
|
320
|
+
"pane_current_path": "/home/user/project/apps/api",
|
|
223
321
|
"window_id": "@1",
|
|
224
322
|
"pane_id": "%1"
|
|
225
323
|
}
|
|
@@ -69,6 +69,67 @@ command = "docker compose up postgres"
|
|
|
69
69
|
auto_start = false
|
|
70
70
|
```
|
|
71
71
|
|
|
72
|
+
## Full Example
|
|
73
|
+
|
|
74
|
+
A full-stack app with a database, API server, and frontend — using health checks to ensure each service is ready before starting its dependents:
|
|
75
|
+
|
|
76
|
+
```toml
|
|
77
|
+
name = "fullstack-app"
|
|
78
|
+
|
|
79
|
+
[tasks.db]
|
|
80
|
+
command = "docker compose up postgres redis"
|
|
81
|
+
health_check = "pg_isready -h localhost -p 5432"
|
|
82
|
+
health_interval = 3
|
|
83
|
+
|
|
84
|
+
[tasks.migrate]
|
|
85
|
+
command = "python manage.py migrate && echo 'done' && sleep infinity"
|
|
86
|
+
cwd = "apps/api"
|
|
87
|
+
depends_on = ["db"]
|
|
88
|
+
health_check = "test -f .migrate-complete"
|
|
89
|
+
|
|
90
|
+
[tasks.api]
|
|
91
|
+
command = "python manage.py runserver 0.0.0.0:8000"
|
|
92
|
+
cwd = "apps/api"
|
|
93
|
+
depends_on = ["migrate"]
|
|
94
|
+
health_check = "curl -sf http://localhost:8000/health"
|
|
95
|
+
|
|
96
|
+
[tasks.worker]
|
|
97
|
+
command = "celery -A myapp worker -l info"
|
|
98
|
+
cwd = "apps/api"
|
|
99
|
+
depends_on = ["db"]
|
|
100
|
+
|
|
101
|
+
[tasks.web]
|
|
102
|
+
command = "bun dev"
|
|
103
|
+
cwd = "apps/web"
|
|
104
|
+
depends_on = ["api"]
|
|
105
|
+
health_check = "curl -sf http://localhost:3000"
|
|
106
|
+
|
|
107
|
+
[tasks.storybook]
|
|
108
|
+
command = "bun storybook"
|
|
109
|
+
cwd = "apps/web"
|
|
110
|
+
auto_start = false
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
What happens on `taskmux start`:
|
|
114
|
+
|
|
115
|
+
1. **db** starts first (no dependencies)
|
|
116
|
+
2. **migrate** and **worker** wait for db's health check (`pg_isready`) to pass
|
|
117
|
+
3. **api** waits for migrate's health check
|
|
118
|
+
4. **web** waits for api's health check (`curl localhost:8000/health`)
|
|
119
|
+
5. **storybook** is skipped (`auto_start = false`) — start it manually with `taskmux start storybook`
|
|
120
|
+
|
|
121
|
+
```bash
|
|
122
|
+
taskmux start # Starts everything in dependency order
|
|
123
|
+
taskmux logs # Interleaved logs from all tasks
|
|
124
|
+
taskmux logs -g "ERROR" # Grep all tasks for errors
|
|
125
|
+
taskmux logs api # Logs from just the API
|
|
126
|
+
taskmux logs -f api # Follow API logs live
|
|
127
|
+
taskmux health # Health check table
|
|
128
|
+
taskmux inspect api # JSON state for a single task
|
|
129
|
+
taskmux restart worker # Restart just the worker
|
|
130
|
+
taskmux start storybook # Start a manual task
|
|
131
|
+
```
|
|
132
|
+
|
|
72
133
|
## Commands
|
|
73
134
|
|
|
74
135
|
```bash
|
|
@@ -89,10 +150,13 @@ taskmux remove <task> # Remove task from config
|
|
|
89
150
|
taskmux inspect <task> # JSON task state (pid, command, health)
|
|
90
151
|
|
|
91
152
|
# Logs
|
|
92
|
-
taskmux logs
|
|
93
|
-
taskmux logs
|
|
94
|
-
taskmux logs -
|
|
95
|
-
taskmux logs <task>
|
|
153
|
+
taskmux logs # Interleaved logs from all tasks
|
|
154
|
+
taskmux logs <task> # Show recent logs for a task
|
|
155
|
+
taskmux logs -f # Attach to session (switch windows with tmux keybinds)
|
|
156
|
+
taskmux logs -f <task> # Follow a task's logs live
|
|
157
|
+
taskmux logs -n 200 <task> # Last N lines
|
|
158
|
+
taskmux logs -g "error" # Search all tasks
|
|
159
|
+
taskmux logs <task> -g "error" # Search one task
|
|
96
160
|
taskmux logs <task> -g "error" -C 5 # Grep with context lines
|
|
97
161
|
|
|
98
162
|
# Init
|
|
@@ -126,12 +190,20 @@ after_stop = "echo done"
|
|
|
126
190
|
|
|
127
191
|
[tasks.server]
|
|
128
192
|
command = "python manage.py runserver"
|
|
193
|
+
cwd = "apps/api"
|
|
194
|
+
health_check = "curl -sf http://localhost:8000/health"
|
|
195
|
+
depends_on = ["db"]
|
|
129
196
|
|
|
130
197
|
[tasks.server.hooks]
|
|
131
198
|
before_start = "python manage.py migrate"
|
|
132
199
|
|
|
200
|
+
[tasks.db]
|
|
201
|
+
command = "docker compose up postgres"
|
|
202
|
+
health_check = "pg_isready -h localhost"
|
|
203
|
+
|
|
133
204
|
[tasks.worker]
|
|
134
205
|
command = "celery worker -A myapp"
|
|
206
|
+
depends_on = ["db"]
|
|
135
207
|
|
|
136
208
|
[tasks.tailwind]
|
|
137
209
|
command = "npx tailwindcss -w"
|
|
@@ -150,8 +222,31 @@ auto_start = false
|
|
|
150
222
|
| `hooks.after_stop` | — | Run after stopping tasks |
|
|
151
223
|
| `tasks.<name>.command` | — | Shell command to run |
|
|
152
224
|
| `tasks.<name>.auto_start` | `true` | Start with `taskmux start` |
|
|
225
|
+
| `tasks.<name>.cwd` | — | Working directory for the task |
|
|
226
|
+
| `tasks.<name>.health_check` | — | Shell command to check health (exit 0 = healthy) |
|
|
227
|
+
| `tasks.<name>.health_interval` | `10` | Seconds between health checks |
|
|
228
|
+
| `tasks.<name>.health_timeout` | `5` | Seconds before health check times out |
|
|
229
|
+
| `tasks.<name>.health_retries` | `3` | Consecutive failures before "unhealthy" |
|
|
230
|
+
| `tasks.<name>.depends_on` | `[]` | Task names that must be healthy before this task starts |
|
|
153
231
|
| `tasks.<name>.hooks.*` | — | Per-task lifecycle hooks (same fields as global) |
|
|
154
232
|
|
|
233
|
+
### Dependency Ordering
|
|
234
|
+
|
|
235
|
+
Tasks with `depends_on` are started in topological order. Before starting a task, taskmux waits for each dependency's health check to pass (up to `health_retries * health_interval` seconds). If a dependency never becomes healthy, the dependent task is skipped with a warning.
|
|
236
|
+
|
|
237
|
+
Circular dependencies and references to nonexistent tasks are rejected at config load time.
|
|
238
|
+
|
|
239
|
+
When starting a single task with `taskmux start <task>`, dependencies are not auto-started — you get a warning if they aren't running.
|
|
240
|
+
|
|
241
|
+
### Health Checks
|
|
242
|
+
|
|
243
|
+
If `health_check` is set, taskmux runs it as a shell command. Exit code 0 means healthy. If not set, taskmux falls back to checking if the tmux pane has a running process (not just a shell prompt).
|
|
244
|
+
|
|
245
|
+
Health checks are used by:
|
|
246
|
+
- `taskmux health` — shows a table of all task health
|
|
247
|
+
- `taskmux start` — waits for dependencies to be healthy before starting dependents
|
|
248
|
+
- `taskmux daemon` — continuously monitors and auto-restarts unhealthy tasks
|
|
249
|
+
|
|
155
250
|
### Hook Cascade
|
|
156
251
|
|
|
157
252
|
Hooks fire in this order:
|
|
@@ -177,14 +272,17 @@ Use `--defaults` to skip prompts (CI/automation).
|
|
|
177
272
|
|
|
178
273
|
```json
|
|
179
274
|
{
|
|
180
|
-
"name": "
|
|
181
|
-
"command": "
|
|
275
|
+
"name": "api",
|
|
276
|
+
"command": "python manage.py runserver 0.0.0.0:8000",
|
|
182
277
|
"auto_start": true,
|
|
278
|
+
"cwd": "apps/api",
|
|
279
|
+
"health_check": "curl -sf http://localhost:8000/health",
|
|
280
|
+
"depends_on": ["db"],
|
|
183
281
|
"running": true,
|
|
184
282
|
"healthy": true,
|
|
185
283
|
"pid": "12345",
|
|
186
|
-
"pane_current_command": "
|
|
187
|
-
"pane_current_path": "/home/user/project",
|
|
284
|
+
"pane_current_command": "python",
|
|
285
|
+
"pane_current_path": "/home/user/project/apps/api",
|
|
188
286
|
"window_id": "@1",
|
|
189
287
|
"pane_id": "%1"
|
|
190
288
|
}
|
|
@@ -106,6 +106,8 @@ def writeConfig(path: Path | None, config: TaskmuxConfig) -> Path:
|
|
|
106
106
|
inner.add("auto_start", False)
|
|
107
107
|
if task_cfg.cwd is not None:
|
|
108
108
|
inner.add("cwd", task_cfg.cwd)
|
|
109
|
+
if task_cfg.port is not None:
|
|
110
|
+
inner.add("port", task_cfg.port)
|
|
109
111
|
if task_cfg.health_check is not None:
|
|
110
112
|
inner.add("health_check", task_cfg.health_check)
|
|
111
113
|
if task_cfg.health_interval != 10:
|
|
@@ -114,6 +116,12 @@ def writeConfig(path: Path | None, config: TaskmuxConfig) -> Path:
|
|
|
114
116
|
inner.add("health_timeout", task_cfg.health_timeout)
|
|
115
117
|
if task_cfg.health_retries != 3:
|
|
116
118
|
inner.add("health_retries", task_cfg.health_retries)
|
|
119
|
+
if task_cfg.stop_grace_period != 5:
|
|
120
|
+
inner.add("stop_grace_period", task_cfg.stop_grace_period)
|
|
121
|
+
if task_cfg.max_restarts != 5:
|
|
122
|
+
inner.add("max_restarts", task_cfg.max_restarts)
|
|
123
|
+
if task_cfg.restart_backoff != 2.0:
|
|
124
|
+
inner.add("restart_backoff", task_cfg.restart_backoff)
|
|
117
125
|
if task_cfg.depends_on:
|
|
118
126
|
inner.add("depends_on", task_cfg.depends_on)
|
|
119
127
|
# Task-level hooks
|
|
@@ -39,6 +39,26 @@ class ConfigWatcher(FileSystemEventHandler):
|
|
|
39
39
|
self.taskmux_cli.handle_config_reload()
|
|
40
40
|
|
|
41
41
|
|
|
42
|
+
class RestartTracker:
|
|
43
|
+
"""Tracks per-task restart counts and timestamps for backoff."""
|
|
44
|
+
|
|
45
|
+
def __init__(self) -> None:
|
|
46
|
+
self._data: dict[str, dict[str, float]] = {}
|
|
47
|
+
|
|
48
|
+
def get(self, task_name: str) -> dict[str, float]:
|
|
49
|
+
return self._data.get(task_name, {"count": 0, "last": 0.0})
|
|
50
|
+
|
|
51
|
+
def record(self, task_name: str) -> None:
|
|
52
|
+
info = self.get(task_name)
|
|
53
|
+
self._data[task_name] = {
|
|
54
|
+
"count": info["count"] + 1,
|
|
55
|
+
"last": time.time(),
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
def reset(self, task_name: str) -> None:
|
|
59
|
+
self._data.pop(task_name, None)
|
|
60
|
+
|
|
61
|
+
|
|
42
62
|
class TaskmuxDaemon:
|
|
43
63
|
"""Daemon mode for Taskmux with enhanced monitoring and API"""
|
|
44
64
|
|
|
@@ -51,6 +71,7 @@ class TaskmuxDaemon:
|
|
|
51
71
|
self.health_check_interval = 30
|
|
52
72
|
self.health_check_task: asyncio.Task | None = None
|
|
53
73
|
self.websocket_clients: set = set()
|
|
74
|
+
self.restart_tracker = RestartTracker()
|
|
54
75
|
self.logger = self._setup_logging()
|
|
55
76
|
|
|
56
77
|
signal.signal(signal.SIGINT, self._signal_handler)
|
|
@@ -124,11 +145,11 @@ class TaskmuxDaemon:
|
|
|
124
145
|
self.logger.info("Taskmux daemon stopped")
|
|
125
146
|
|
|
126
147
|
async def _health_check_loop(self) -> None:
|
|
127
|
-
"""Continuous health checking loop"""
|
|
148
|
+
"""Continuous health checking loop with restart backoff."""
|
|
128
149
|
while self.running:
|
|
129
150
|
try:
|
|
130
151
|
if self.cli and self.cli.tmux.session_exists():
|
|
131
|
-
self.
|
|
152
|
+
self._auto_restart_with_backoff()
|
|
132
153
|
|
|
133
154
|
if self.websocket_clients:
|
|
134
155
|
status = await self._get_full_status()
|
|
@@ -139,6 +160,42 @@ class TaskmuxDaemon:
|
|
|
139
160
|
self.logger.error(f"Health check error: {e}")
|
|
140
161
|
await asyncio.sleep(5)
|
|
141
162
|
|
|
163
|
+
def _auto_restart_with_backoff(self) -> None:
|
|
164
|
+
"""Auto-restart unhealthy tasks with exponential backoff."""
|
|
165
|
+
assert self.cli is not None
|
|
166
|
+
now = time.time()
|
|
167
|
+
|
|
168
|
+
for task_name, task_cfg in self.cli.config.tasks.items():
|
|
169
|
+
healthy = self.cli.tmux.check_task_health(task_name)
|
|
170
|
+
|
|
171
|
+
if healthy:
|
|
172
|
+
# Reset tracker if healthy for >60s
|
|
173
|
+
info = self.restart_tracker.get(task_name)
|
|
174
|
+
if info["count"] > 0 and now - info["last"] > 60:
|
|
175
|
+
self.restart_tracker.reset(task_name)
|
|
176
|
+
continue
|
|
177
|
+
|
|
178
|
+
# Skip if not previously healthy (avoid restart loop on first check)
|
|
179
|
+
prev_health = self.cli.tmux.task_health.get(task_name, {}).get("healthy", True)
|
|
180
|
+
if not prev_health:
|
|
181
|
+
info = self.restart_tracker.get(task_name)
|
|
182
|
+
|
|
183
|
+
# Check max_restarts
|
|
184
|
+
if task_cfg.max_restarts and info["count"] >= task_cfg.max_restarts:
|
|
185
|
+
self.logger.warning(
|
|
186
|
+
f"Task '{task_name}' exceeded max restarts ({task_cfg.max_restarts})"
|
|
187
|
+
)
|
|
188
|
+
continue
|
|
189
|
+
|
|
190
|
+
# Check backoff delay
|
|
191
|
+
delay = min(task_cfg.restart_backoff ** info["count"], 60)
|
|
192
|
+
if info["last"] and now - info["last"] < delay:
|
|
193
|
+
continue
|
|
194
|
+
|
|
195
|
+
self.logger.info(f"Auto-restarting unhealthy task: {task_name}")
|
|
196
|
+
self.cli.tmux.restart_task(task_name)
|
|
197
|
+
self.restart_tracker.record(task_name)
|
|
198
|
+
|
|
142
199
|
async def _start_api_server(self) -> None:
|
|
143
200
|
"""Start WebSocket API server"""
|
|
144
201
|
|
|
@@ -37,10 +37,14 @@ class TaskConfig(_StrictConfig):
|
|
|
37
37
|
command: str
|
|
38
38
|
auto_start: bool = True
|
|
39
39
|
cwd: str | None = None
|
|
40
|
+
port: int | None = None
|
|
40
41
|
health_check: str | None = None
|
|
41
42
|
health_interval: int = 10
|
|
42
43
|
health_timeout: int = 5
|
|
43
44
|
health_retries: int = 3
|
|
45
|
+
stop_grace_period: int = 5
|
|
46
|
+
max_restarts: int = 5
|
|
47
|
+
restart_backoff: float = 2.0
|
|
44
48
|
depends_on: list[str] = []
|
|
45
49
|
hooks: HookConfig = HookConfig()
|
|
46
50
|
|
|
@@ -2,16 +2,38 @@
|
|
|
2
2
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
|
+
import contextlib
|
|
6
|
+
import os
|
|
7
|
+
import signal as sig
|
|
5
8
|
import subprocess
|
|
6
9
|
import time
|
|
7
10
|
from collections import deque
|
|
8
11
|
from datetime import datetime
|
|
9
12
|
|
|
10
13
|
import libtmux
|
|
14
|
+
from rich.console import Console
|
|
15
|
+
from rich.markup import escape
|
|
11
16
|
|
|
12
17
|
from .hooks import runHook
|
|
13
18
|
from .models import TaskmuxConfig
|
|
14
19
|
|
|
20
|
+
SHELL_NAMES = frozenset(("bash", "zsh", "sh", "fish"))
|
|
21
|
+
|
|
22
|
+
TASK_COLORS = ["cyan", "green", "yellow", "magenta", "blue", "red"]
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _find_new_lines(current: list[str], prev_tail: list[str]) -> list[str]:
|
|
26
|
+
"""Return lines in current that are new since prev_tail."""
|
|
27
|
+
if not prev_tail:
|
|
28
|
+
return current
|
|
29
|
+
target = prev_tail[-1]
|
|
30
|
+
for i in range(len(current) - 1, -1, -1):
|
|
31
|
+
if current[i] == target:
|
|
32
|
+
ctx = min(len(prev_tail), i + 1)
|
|
33
|
+
if current[i - ctx + 1 : i + 1] == prev_tail[-ctx:]:
|
|
34
|
+
return current[i + 1 :]
|
|
35
|
+
return current # no match, prev scrolled away — return all
|
|
36
|
+
|
|
15
37
|
|
|
16
38
|
class TmuxManager:
|
|
17
39
|
"""Manages tmux sessions and tasks using libtmux API."""
|
|
@@ -57,11 +79,61 @@ class TmuxManager:
|
|
|
57
79
|
window = self._get_session().windows.get(window_name=task_name, default=None)
|
|
58
80
|
if window and window.active_pane:
|
|
59
81
|
cmd = getattr(window.active_pane, "pane_current_command", "")
|
|
60
|
-
return cmd != "" and cmd
|
|
82
|
+
return cmd != "" and cmd not in SHELL_NAMES
|
|
61
83
|
except Exception:
|
|
62
84
|
pass
|
|
63
85
|
return False
|
|
64
86
|
|
|
87
|
+
def _wait_for_exit(self, pane: libtmux.Pane, timeout: float) -> bool:
|
|
88
|
+
"""Poll pane_current_command until it returns to a shell or timeout."""
|
|
89
|
+
elapsed = 0.0
|
|
90
|
+
while elapsed < timeout:
|
|
91
|
+
time.sleep(0.5)
|
|
92
|
+
elapsed += 0.5
|
|
93
|
+
cmd = getattr(pane, "pane_current_command", "")
|
|
94
|
+
if cmd == "" or cmd in SHELL_NAMES:
|
|
95
|
+
return True
|
|
96
|
+
return False
|
|
97
|
+
|
|
98
|
+
def _get_pane_child_pid(self, pane: libtmux.Pane) -> int | None:
|
|
99
|
+
"""Get the child process PID running inside the pane's shell."""
|
|
100
|
+
shell_pid = getattr(pane, "pane_pid", None)
|
|
101
|
+
if not shell_pid:
|
|
102
|
+
return None
|
|
103
|
+
try:
|
|
104
|
+
result = subprocess.run(
|
|
105
|
+
["pgrep", "-P", str(shell_pid)],
|
|
106
|
+
capture_output=True,
|
|
107
|
+
text=True,
|
|
108
|
+
)
|
|
109
|
+
pids = result.stdout.strip().split("\n")
|
|
110
|
+
return int(pids[0]) if pids and pids[0] else None
|
|
111
|
+
except (ValueError, OSError):
|
|
112
|
+
return None
|
|
113
|
+
|
|
114
|
+
def _kill_process_tree(self, pid: int, signal_num: int = sig.SIGKILL) -> None:
|
|
115
|
+
"""Kill process and all children via process group."""
|
|
116
|
+
try:
|
|
117
|
+
pgid = os.getpgid(pid)
|
|
118
|
+
os.killpg(pgid, signal_num)
|
|
119
|
+
except (ProcessLookupError, PermissionError, OSError):
|
|
120
|
+
pass
|
|
121
|
+
|
|
122
|
+
def _cleanup_port(self, port: int) -> None:
|
|
123
|
+
"""Kill any process listening on port."""
|
|
124
|
+
try:
|
|
125
|
+
result = subprocess.run(
|
|
126
|
+
["lsof", "-ti", f":{port}"],
|
|
127
|
+
capture_output=True,
|
|
128
|
+
text=True,
|
|
129
|
+
)
|
|
130
|
+
for pid_str in result.stdout.strip().split("\n"):
|
|
131
|
+
if pid_str.strip():
|
|
132
|
+
with contextlib.suppress(ProcessLookupError, PermissionError, OSError):
|
|
133
|
+
os.kill(int(pid_str.strip()), sig.SIGKILL)
|
|
134
|
+
except OSError:
|
|
135
|
+
pass
|
|
136
|
+
|
|
65
137
|
def is_task_healthy(self, task_name: str) -> bool:
|
|
66
138
|
"""Check task health. Uses health_check command if configured, falls back to pane-alive."""
|
|
67
139
|
task_cfg = self.config.tasks.get(task_name)
|
|
@@ -171,6 +243,10 @@ class TmuxManager:
|
|
|
171
243
|
sess = self._get_session()
|
|
172
244
|
task_cfg = self.config.tasks[task_name]
|
|
173
245
|
|
|
246
|
+
# Kill anything occupying the port before starting
|
|
247
|
+
if task_cfg.port:
|
|
248
|
+
self._cleanup_port(task_cfg.port)
|
|
249
|
+
|
|
174
250
|
# Check if already running
|
|
175
251
|
existing = sess.windows.get(window_name=task_name, default=None)
|
|
176
252
|
if existing:
|
|
@@ -209,7 +285,7 @@ class TmuxManager:
|
|
|
209
285
|
print(f"Started task '{task_name}'")
|
|
210
286
|
|
|
211
287
|
def stop_task(self, task_name: str) -> None:
|
|
212
|
-
"""Graceful stop
|
|
288
|
+
"""Graceful stop with signal escalation: C-c → SIGTERM → SIGKILL."""
|
|
213
289
|
if not self.session_exists():
|
|
214
290
|
print(f"Session '{self.config.name}' doesn't exist")
|
|
215
291
|
return
|
|
@@ -232,8 +308,22 @@ class TmuxManager:
|
|
|
232
308
|
|
|
233
309
|
pane = window.active_pane
|
|
234
310
|
if pane:
|
|
311
|
+
# Phase 1: SIGINT (Ctrl+C)
|
|
235
312
|
pane.send_keys("C-c")
|
|
236
313
|
|
|
314
|
+
if not self._wait_for_exit(pane, timeout=task_cfg.stop_grace_period):
|
|
315
|
+
# Phase 2: SIGTERM via process group
|
|
316
|
+
pid = self._get_pane_child_pid(pane)
|
|
317
|
+
if pid:
|
|
318
|
+
self._kill_process_tree(pid, sig.SIGTERM)
|
|
319
|
+
|
|
320
|
+
if not self._wait_for_exit(pane, timeout=3):
|
|
321
|
+
# Phase 3: SIGKILL entire process group
|
|
322
|
+
if pid:
|
|
323
|
+
self._kill_process_tree(pid, sig.SIGKILL)
|
|
324
|
+
# Final wait for cleanup
|
|
325
|
+
self._wait_for_exit(pane, timeout=1)
|
|
326
|
+
|
|
237
327
|
# Hooks: task after_stop, then global after_stop
|
|
238
328
|
runHook(task_cfg.hooks.after_stop, task_name)
|
|
239
329
|
runHook(self.config.hooks.after_stop, task_name)
|
|
@@ -305,7 +395,7 @@ class TmuxManager:
|
|
|
305
395
|
print(f"Started session '{self.config.name}' with {len(auto_tasks)} tasks")
|
|
306
396
|
|
|
307
397
|
def stop_all(self) -> None:
|
|
308
|
-
"""Stop all tasks then kill session."""
|
|
398
|
+
"""Stop all tasks with signal escalation then kill session."""
|
|
309
399
|
if not self.session_exists():
|
|
310
400
|
print("No session running")
|
|
311
401
|
return
|
|
@@ -313,8 +403,9 @@ class TmuxManager:
|
|
|
313
403
|
# Global before_stop
|
|
314
404
|
runHook(self.config.hooks.before_stop)
|
|
315
405
|
|
|
316
|
-
#
|
|
406
|
+
# Phase 1: send C-c to all tasks
|
|
317
407
|
sess = self._get_session()
|
|
408
|
+
pane_map: dict[str, tuple[libtmux.Pane, int | None]] = {}
|
|
318
409
|
for task_name, task_cfg in self.config.tasks.items():
|
|
319
410
|
window = sess.windows.get(window_name=task_name, default=None)
|
|
320
411
|
if window:
|
|
@@ -322,7 +413,30 @@ class TmuxManager:
|
|
|
322
413
|
pane = window.active_pane
|
|
323
414
|
if pane:
|
|
324
415
|
pane.send_keys("C-c")
|
|
325
|
-
|
|
416
|
+
pid = self._get_pane_child_pid(pane)
|
|
417
|
+
pane_map[task_name] = (pane, pid)
|
|
418
|
+
|
|
419
|
+
# Wait for graceful exit (use max grace period across tasks)
|
|
420
|
+
max_grace = max((cfg.stop_grace_period for cfg in self.config.tasks.values()), default=5)
|
|
421
|
+
time.sleep(max_grace)
|
|
422
|
+
|
|
423
|
+
# Phase 2: SIGTERM then SIGKILL any survivors
|
|
424
|
+
for _name, (pane, pid) in pane_map.items():
|
|
425
|
+
cmd = getattr(pane, "pane_current_command", "")
|
|
426
|
+
if cmd and cmd not in SHELL_NAMES and pid:
|
|
427
|
+
self._kill_process_tree(pid, sig.SIGTERM)
|
|
428
|
+
|
|
429
|
+
time.sleep(1)
|
|
430
|
+
|
|
431
|
+
for _name, (pane, pid) in pane_map.items():
|
|
432
|
+
cmd = getattr(pane, "pane_current_command", "")
|
|
433
|
+
if cmd and cmd not in SHELL_NAMES and pid:
|
|
434
|
+
self._kill_process_tree(pid, sig.SIGKILL)
|
|
435
|
+
|
|
436
|
+
# Run after_stop hooks
|
|
437
|
+
for task_name in pane_map:
|
|
438
|
+
task_cfg = self.config.tasks[task_name]
|
|
439
|
+
runHook(task_cfg.hooks.after_stop, task_name)
|
|
326
440
|
|
|
327
441
|
sess.kill()
|
|
328
442
|
|
|
@@ -341,7 +455,7 @@ class TmuxManager:
|
|
|
341
455
|
self.start_all()
|
|
342
456
|
|
|
343
457
|
def restart_task(self, task_name: str) -> None:
|
|
344
|
-
"""Restart a specific task
|
|
458
|
+
"""Restart a specific task with full stop escalation."""
|
|
345
459
|
if not self.session_exists():
|
|
346
460
|
print(f"Session '{self.config.name}' doesn't exist. Run 'taskmux start' first.")
|
|
347
461
|
return
|
|
@@ -356,13 +470,25 @@ class TmuxManager:
|
|
|
356
470
|
|
|
357
471
|
window = sess.windows.get(window_name=task_name, default=None)
|
|
358
472
|
if window:
|
|
473
|
+
# Full stop with signal escalation
|
|
359
474
|
runHook(task_cfg.hooks.before_stop, task_name)
|
|
360
475
|
pane = window.active_pane
|
|
361
476
|
if pane:
|
|
362
477
|
pane.send_keys("C-c")
|
|
363
|
-
|
|
478
|
+
if not self._wait_for_exit(pane, timeout=task_cfg.stop_grace_period):
|
|
479
|
+
pid = self._get_pane_child_pid(pane)
|
|
480
|
+
if pid:
|
|
481
|
+
self._kill_process_tree(pid, sig.SIGTERM)
|
|
482
|
+
if not self._wait_for_exit(pane, timeout=3):
|
|
483
|
+
if pid:
|
|
484
|
+
self._kill_process_tree(pid, sig.SIGKILL)
|
|
485
|
+
self._wait_for_exit(pane, timeout=1)
|
|
364
486
|
runHook(task_cfg.hooks.after_stop, task_name)
|
|
365
487
|
|
|
488
|
+
# Port cleanup before restart
|
|
489
|
+
if task_cfg.port:
|
|
490
|
+
self._cleanup_port(task_cfg.port)
|
|
491
|
+
|
|
366
492
|
runHook(task_cfg.hooks.before_start, task_name)
|
|
367
493
|
pane = window.active_pane
|
|
368
494
|
if pane:
|
|
@@ -371,6 +497,9 @@ class TmuxManager:
|
|
|
371
497
|
pane.send_keys(command, enter=True)
|
|
372
498
|
runHook(task_cfg.hooks.after_start, task_name)
|
|
373
499
|
else:
|
|
500
|
+
# Port cleanup before start
|
|
501
|
+
if task_cfg.port:
|
|
502
|
+
self._cleanup_port(task_cfg.port)
|
|
374
503
|
runHook(task_cfg.hooks.before_start, task_name)
|
|
375
504
|
self._send_command_to_window(sess, task_name, command, task_cfg.cwd)
|
|
376
505
|
runHook(task_cfg.hooks.after_start, task_name)
|
|
@@ -378,13 +507,18 @@ class TmuxManager:
|
|
|
378
507
|
print(f"Restarted task '{task_name}'")
|
|
379
508
|
|
|
380
509
|
def kill_task(self, task_name: str) -> None:
|
|
381
|
-
"""Kill a specific task"""
|
|
510
|
+
"""Kill a specific task (process group + window)."""
|
|
382
511
|
if not self.session_exists():
|
|
383
512
|
print(f"Session '{self.config.name}' doesn't exist")
|
|
384
513
|
return
|
|
385
514
|
|
|
386
515
|
window = self._get_session().windows.get(window_name=task_name, default=None)
|
|
387
516
|
if window:
|
|
517
|
+
pane = window.active_pane
|
|
518
|
+
if pane:
|
|
519
|
+
pid = self._get_pane_child_pid(pane)
|
|
520
|
+
if pid:
|
|
521
|
+
self._kill_process_tree(pid)
|
|
388
522
|
window.kill()
|
|
389
523
|
print(f"Killed task '{task_name}'")
|
|
390
524
|
else:
|
|
@@ -433,6 +567,54 @@ class TmuxManager:
|
|
|
433
567
|
info["healthy"] = self.is_task_healthy(task_name)
|
|
434
568
|
return info
|
|
435
569
|
|
|
570
|
+
def _tail_panes(
|
|
571
|
+
self,
|
|
572
|
+
panes: list[tuple[str, libtmux.Pane, str]],
|
|
573
|
+
lines: int = 100,
|
|
574
|
+
grep: str | None = None,
|
|
575
|
+
) -> None:
|
|
576
|
+
"""Poll capture-pane and print new lines with colored task prefixes."""
|
|
577
|
+
console = Console()
|
|
578
|
+
state: dict[str, list[str]] = {}
|
|
579
|
+
|
|
580
|
+
try:
|
|
581
|
+
while True:
|
|
582
|
+
for task_name, pane, color in panes:
|
|
583
|
+
output = pane.cmd("capture-pane", "-p", "-S", f"-{lines}").stdout
|
|
584
|
+
while output and not output[-1].strip():
|
|
585
|
+
output.pop()
|
|
586
|
+
|
|
587
|
+
prev = state.get(task_name, [])
|
|
588
|
+
new = _find_new_lines(output, prev)
|
|
589
|
+
|
|
590
|
+
if grep:
|
|
591
|
+
new = [ln for ln in new if grep.lower() in ln.lower()]
|
|
592
|
+
|
|
593
|
+
for line in new:
|
|
594
|
+
prefix = escape(f"[{task_name}]")
|
|
595
|
+
console.print(f"[{color}]{prefix}[/{color}] {escape(line)}")
|
|
596
|
+
|
|
597
|
+
if output:
|
|
598
|
+
state[task_name] = output[-50:]
|
|
599
|
+
|
|
600
|
+
time.sleep(0.5)
|
|
601
|
+
except KeyboardInterrupt:
|
|
602
|
+
console.print("\n[dim]Stopped following logs[/dim]")
|
|
603
|
+
|
|
604
|
+
def _collect_panes(self, task_names: list[str]) -> list[tuple[str, libtmux.Pane, str]]:
|
|
605
|
+
"""Collect (name, pane, color) tuples for running tasks."""
|
|
606
|
+
sess = self._get_session()
|
|
607
|
+
result: list[tuple[str, libtmux.Pane, str]] = []
|
|
608
|
+
for i, name in enumerate(task_names):
|
|
609
|
+
window = sess.windows.get(window_name=name, default=None)
|
|
610
|
+
if not window:
|
|
611
|
+
continue
|
|
612
|
+
pane = window.active_pane
|
|
613
|
+
if pane:
|
|
614
|
+
color = TASK_COLORS[i % len(TASK_COLORS)]
|
|
615
|
+
result.append((name, pane, color))
|
|
616
|
+
return result
|
|
617
|
+
|
|
436
618
|
def show_logs(
|
|
437
619
|
self,
|
|
438
620
|
task_name: str | None,
|
|
@@ -461,8 +643,9 @@ class TmuxManager:
|
|
|
461
643
|
return
|
|
462
644
|
|
|
463
645
|
if follow:
|
|
464
|
-
|
|
465
|
-
|
|
646
|
+
panes = self._collect_panes([task_name])
|
|
647
|
+
if panes:
|
|
648
|
+
self._tail_panes(panes, lines=lines, grep=grep)
|
|
466
649
|
else:
|
|
467
650
|
pane = window.active_pane
|
|
468
651
|
if pane:
|
|
@@ -482,26 +665,33 @@ class TmuxManager:
|
|
|
482
665
|
) -> None:
|
|
483
666
|
"""Show logs from all running tasks."""
|
|
484
667
|
sess = self._get_session()
|
|
668
|
+
console = Console()
|
|
669
|
+
task_names = list(self.config.tasks.keys())
|
|
485
670
|
|
|
486
671
|
if follow:
|
|
487
|
-
|
|
672
|
+
panes = self._collect_panes(task_names)
|
|
673
|
+
if panes:
|
|
674
|
+
self._tail_panes(panes, lines=lines, grep=grep)
|
|
488
675
|
return
|
|
489
676
|
|
|
490
|
-
for task_name in
|
|
677
|
+
for i, task_name in enumerate(task_names):
|
|
491
678
|
window = sess.windows.get(window_name=task_name, default=None)
|
|
492
679
|
if not window:
|
|
493
680
|
continue
|
|
494
681
|
pane = window.active_pane
|
|
495
682
|
if not pane:
|
|
496
683
|
continue
|
|
684
|
+
color = TASK_COLORS[i % len(TASK_COLORS)]
|
|
497
685
|
output = pane.cmd("capture-pane", "-p", "-S", f"-{lines}").stdout
|
|
498
686
|
if grep:
|
|
499
687
|
matching = [line for line in output if grep.lower() in line.lower()]
|
|
500
688
|
for line in matching:
|
|
501
|
-
|
|
689
|
+
prefix = escape(f"[{task_name}]")
|
|
690
|
+
console.print(f"[{color}]{prefix}[/{color}] {escape(line)}")
|
|
502
691
|
else:
|
|
503
692
|
for line in output:
|
|
504
|
-
|
|
693
|
+
prefix = escape(f"[{task_name}]")
|
|
694
|
+
console.print(f"[{color}]{prefix}[/{color}] {escape(line)}")
|
|
505
695
|
|
|
506
696
|
def list_tasks(self) -> None:
|
|
507
697
|
"""List all tasks and their status"""
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|