provide-uterm-platform 0.5.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- provide_uterm_platform-0.5.0/PKG-INFO +38 -0
- provide_uterm_platform-0.5.0/VERSION +1 -0
- provide_uterm_platform-0.5.0/pyproject.toml +120 -0
- provide_uterm_platform-0.5.0/setup.cfg +4 -0
- provide_uterm_platform-0.5.0/src/provide/uterm/manager/__init__.py +33 -0
- provide_uterm_platform-0.5.0/src/provide/uterm/manager/_monitor.py +271 -0
- provide_uterm_platform-0.5.0/src/provide/uterm/manager/app.py +154 -0
- provide_uterm_platform-0.5.0/src/provide/uterm/manager/auth.py +296 -0
- provide_uterm_platform-0.5.0/src/provide/uterm/manager/cli.py +41 -0
- provide_uterm_platform-0.5.0/src/provide/uterm/manager/config.py +85 -0
- provide_uterm_platform-0.5.0/src/provide/uterm/manager/constants.py +57 -0
- provide_uterm_platform-0.5.0/src/provide/uterm/manager/core.py +414 -0
- provide_uterm_platform-0.5.0/src/provide/uterm/manager/ext.py +68 -0
- provide_uterm_platform-0.5.0/src/provide/uterm/manager/mcp_tools.py +346 -0
- provide_uterm_platform-0.5.0/src/provide/uterm/manager/models.py +110 -0
- provide_uterm_platform-0.5.0/src/provide/uterm/manager/process.py +27 -0
- provide_uterm_platform-0.5.0/src/provide/uterm/manager/process_impl.py +491 -0
- provide_uterm_platform-0.5.0/src/provide/uterm/manager/process_impl_spawn.py +251 -0
- provide_uterm_platform-0.5.0/src/provide/uterm/manager/protocols.py +138 -0
- provide_uterm_platform-0.5.0/src/provide/uterm/manager/routes/__init__.py +14 -0
- provide_uterm_platform-0.5.0/src/provide/uterm/manager/routes/agent_ops.py +479 -0
- provide_uterm_platform-0.5.0/src/provide/uterm/manager/routes/agent_update.py +158 -0
- provide_uterm_platform-0.5.0/src/provide/uterm/manager/routes/models.py +42 -0
- provide_uterm_platform-0.5.0/src/provide/uterm/manager/routes/spawn.py +342 -0
- provide_uterm_platform-0.5.0/src/provide/uterm/manager/routes/status.py +39 -0
- provide_uterm_platform-0.5.0/src/provide/uterm/manager/timeseries/__init__.py +12 -0
- provide_uterm_platform-0.5.0/src/provide/uterm/manager/timeseries/manager.py +220 -0
- provide_uterm_platform-0.5.0/src/provide/uterm/pty/__init__.py +33 -0
- provide_uterm_platform-0.5.0/src/provide/uterm/pty/_build.py +71 -0
- provide_uterm_platform-0.5.0/src/provide/uterm/pty/_validate.py +82 -0
- provide_uterm_platform-0.5.0/src/provide/uterm/pty/capture.py +162 -0
- provide_uterm_platform-0.5.0/src/provide/uterm/pty/capture_connector.py +232 -0
- provide_uterm_platform-0.5.0/src/provide/uterm/pty/connector.py +439 -0
- provide_uterm_platform-0.5.0/src/provide/uterm/pty/pam.py +298 -0
- provide_uterm_platform-0.5.0/src/provide/uterm/pty/pam_listener.py +278 -0
- provide_uterm_platform-0.5.0/src/provide/uterm/pty/py.typed +0 -0
- provide_uterm_platform-0.5.0/src/provide/uterm/pty/socket_utils.py +18 -0
- provide_uterm_platform-0.5.0/src/provide/uterm/pty/uid_map.py +133 -0
- provide_uterm_platform-0.5.0/src/provide/uterm/py.typed +0 -0
- provide_uterm_platform-0.5.0/src/provide_uterm_platform.egg-info/PKG-INFO +38 -0
- provide_uterm_platform-0.5.0/src/provide_uterm_platform.egg-info/SOURCES.txt +44 -0
- provide_uterm_platform-0.5.0/src/provide_uterm_platform.egg-info/dependency_links.txt +1 -0
- provide_uterm_platform-0.5.0/src/provide_uterm_platform.egg-info/entry_points.txt +2 -0
- provide_uterm_platform-0.5.0/src/provide_uterm_platform.egg-info/requires.txt +36 -0
- provide_uterm_platform-0.5.0/src/provide_uterm_platform.egg-info/top_level.txt +1 -0
- provide_uterm_platform-0.5.0/tests/test_manager_mcp_tools_register.py +59 -0
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: provide-uterm-platform
|
|
3
|
+
Version: 0.5.0
|
|
4
|
+
Summary: Platform-specific deployment targets (PTY, PAM, External Management Tier) for provide-uterm
|
|
5
|
+
License-Expression: AGPL-3.0-or-later
|
|
6
|
+
Requires-Python: >=3.11
|
|
7
|
+
Requires-Dist: provide-uterm>=0.5.0
|
|
8
|
+
Provides-Extra: pty
|
|
9
|
+
Requires-Dist: provide-uterm>=0.5.0; extra == "pty"
|
|
10
|
+
Provides-Extra: manager
|
|
11
|
+
Requires-Dist: fastapi>=0.110; extra == "manager"
|
|
12
|
+
Requires-Dist: uvicorn>=0.24; extra == "manager"
|
|
13
|
+
Requires-Dist: pydantic>=2.0; extra == "manager"
|
|
14
|
+
Requires-Dist: pyyaml>=6.0; extra == "manager"
|
|
15
|
+
Requires-Dist: httpx2>=2.10; extra == "manager"
|
|
16
|
+
Provides-Extra: all
|
|
17
|
+
Requires-Dist: provide-uterm>=0.5.0; extra == "all"
|
|
18
|
+
Requires-Dist: fastapi>=0.110; extra == "all"
|
|
19
|
+
Requires-Dist: uvicorn>=0.24; extra == "all"
|
|
20
|
+
Requires-Dist: pydantic>=2.0; extra == "all"
|
|
21
|
+
Requires-Dist: pyyaml>=6.0; extra == "all"
|
|
22
|
+
Requires-Dist: httpx2>=2.10; extra == "all"
|
|
23
|
+
Provides-Extra: dev
|
|
24
|
+
Requires-Dist: pytest>=9.0; extra == "dev"
|
|
25
|
+
Requires-Dist: pytest-asyncio>=1.3; extra == "dev"
|
|
26
|
+
Requires-Dist: pytest-cov>=7.0; extra == "dev"
|
|
27
|
+
Requires-Dist: pytest-timeout>=2.4.0; extra == "dev"
|
|
28
|
+
Requires-Dist: mypy>=1.19; extra == "dev"
|
|
29
|
+
Requires-Dist: psutil>=5.9; extra == "dev"
|
|
30
|
+
Requires-Dist: ruff>=0.15; extra == "dev"
|
|
31
|
+
Requires-Dist: bandit>=1.9; extra == "dev"
|
|
32
|
+
Requires-Dist: provide-uterm>=0.5.0; extra == "dev"
|
|
33
|
+
Requires-Dist: fastapi>=0.110; extra == "dev"
|
|
34
|
+
Requires-Dist: uvicorn>=0.24; extra == "dev"
|
|
35
|
+
Requires-Dist: pydantic>=2.0; extra == "dev"
|
|
36
|
+
Requires-Dist: pyyaml>=6.0; extra == "dev"
|
|
37
|
+
Requires-Dist: httpx2>=2.10; extra == "dev"
|
|
38
|
+
Requires-Dist: mcp>=2.0.0; extra == "dev"
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
0.5.0
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 provide.io llc. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: AGPL-3.0-or-later
|
|
3
|
+
|
|
4
|
+
[build-system]
|
|
5
|
+
requires = ["setuptools>=61.0", "wheel"]
|
|
6
|
+
build-backend = "setuptools.build_meta"
|
|
7
|
+
|
|
8
|
+
[project]
|
|
9
|
+
name = "provide-uterm-platform"
|
|
10
|
+
dynamic = ["version"]
|
|
11
|
+
description = "Platform-specific deployment targets (PTY, PAM, External Management Tier) for provide-uterm"
|
|
12
|
+
requires-python = ">=3.11"
|
|
13
|
+
license = "AGPL-3.0-or-later"
|
|
14
|
+
dependencies = [
|
|
15
|
+
"provide-uterm>=0.5.0",
|
|
16
|
+
]
|
|
17
|
+
|
|
18
|
+
[project.optional-dependencies]
|
|
19
|
+
pty = ["provide-uterm>=0.5.0"]
|
|
20
|
+
# ``httpx2`` is an import-time requirement, not an optional nicety: manager/ext.py
|
|
21
|
+
# imports it at module scope and ext.py is reached eagerly from
|
|
22
|
+
# ``provide.uterm.manager`` via app -> process -> process_impl / _monitor.
|
|
23
|
+
manager = ["fastapi>=0.110", "uvicorn>=0.24", "pydantic>=2.0", "pyyaml>=6.0", "httpx2>=2.10"]
|
|
24
|
+
all = ["provide-uterm>=0.5.0", "fastapi>=0.110", "uvicorn>=0.24", "pydantic>=2.0", "pyyaml>=6.0", "httpx2>=2.10"]
|
|
25
|
+
dev = [
|
|
26
|
+
"pytest>=9.0",
|
|
27
|
+
"pytest-asyncio>=1.3",
|
|
28
|
+
"pytest-cov>=7.0",
|
|
29
|
+
"pytest-timeout>=2.4.0",
|
|
30
|
+
"mypy>=1.19",
|
|
31
|
+
"psutil>=5.9",
|
|
32
|
+
"ruff>=0.15",
|
|
33
|
+
"bandit>=1.9",
|
|
34
|
+
"provide-uterm>=0.5.0",
|
|
35
|
+
"fastapi>=0.110",
|
|
36
|
+
"uvicorn>=0.24",
|
|
37
|
+
"pydantic>=2.0",
|
|
38
|
+
"pyyaml>=6.0",
|
|
39
|
+
"httpx2>=2.10",
|
|
40
|
+
# The manager MCP tests build a real mcp.server.mcpserver.MCPServer to
|
|
41
|
+
# register the tools onto. The production module only needs the structural
|
|
42
|
+
# _ToolRegistrar protocol, so mcp is a test-only dependency.
|
|
43
|
+
"mcp>=2.0.0",
|
|
44
|
+
]
|
|
45
|
+
|
|
46
|
+
[project.scripts]
|
|
47
|
+
uterm-manager = "provide.uterm.manager.cli:main"
|
|
48
|
+
|
|
49
|
+
[tool.setuptools.dynamic]
|
|
50
|
+
version = {file = "VERSION"}
|
|
51
|
+
|
|
52
|
+
[tool.setuptools.packages.find]
|
|
53
|
+
where = ["src"]
|
|
54
|
+
|
|
55
|
+
[tool.setuptools.package-data]
|
|
56
|
+
"provide.uterm" = ["py.typed"]
|
|
57
|
+
"provide.uterm.pty" = ["py.typed", "_native/*"]
|
|
58
|
+
|
|
59
|
+
[tool.pytest.ini_options]
|
|
60
|
+
consider_namespace_packages = true
|
|
61
|
+
asyncio_mode = "auto"
|
|
62
|
+
asyncio_default_fixture_loop_scope = "function"
|
|
63
|
+
testpaths = ["tests"]
|
|
64
|
+
addopts = [
|
|
65
|
+
"--import-mode=importlib",
|
|
66
|
+
"-v",
|
|
67
|
+
"--strict-markers",
|
|
68
|
+
"--strict-config",
|
|
69
|
+
"--tb=short",
|
|
70
|
+
"-ra",
|
|
71
|
+
"--cov=provide.uterm.pty",
|
|
72
|
+
"--cov=provide.uterm.manager",
|
|
73
|
+
"--cov-branch",
|
|
74
|
+
"--cov-report=term-missing",
|
|
75
|
+
"--cov-fail-under=100",
|
|
76
|
+
]
|
|
77
|
+
markers = [
|
|
78
|
+
"requires_pam: requires /etc/pam.d/provide-uterm service config (skip in CI)",
|
|
79
|
+
"requires_pam_auth: requires /etc/pam.d/provide-uterm with working unix_chkpwd (skipped in Docker)",
|
|
80
|
+
"requires_root: requires root or CAP_SETUID to switch users",
|
|
81
|
+
]
|
|
82
|
+
|
|
83
|
+
[tool.coverage.report]
|
|
84
|
+
fail_under = 100
|
|
85
|
+
show_missing = true
|
|
86
|
+
partial_branches = [
|
|
87
|
+
"pragma: no branch",
|
|
88
|
+
"if _libpam is not None:",
|
|
89
|
+
]
|
|
90
|
+
|
|
91
|
+
[tool.ruff]
|
|
92
|
+
target-version = "py311"
|
|
93
|
+
line-length = 120
|
|
94
|
+
|
|
95
|
+
[tool.ruff.lint]
|
|
96
|
+
select = [
|
|
97
|
+
"E", "W", "F", "I", "N", "UP", "B", "C4", "SIM", "TCH", "PTH",
|
|
98
|
+
"DTZ", "ICN", "ISC", "PERF", "PIE", "RET", "RSE", "TID", "S", "ARG", "RUF",
|
|
99
|
+
]
|
|
100
|
+
# Aligned with the workspace-root ruff config in ../../pyproject.toml, except
|
|
101
|
+
# ``S110`` is left non-ignored at the production-code level so individual
|
|
102
|
+
# try/except/pass blocks must justify themselves with ``# noqa: S110``.
|
|
103
|
+
ignore = [
|
|
104
|
+
"E501", "S101", "S104", "S108", "S112", "S311", "S603", "S607",
|
|
105
|
+
"S604", "F811", "N806", "N814", "B017", "SIM105", "SIM117", "ARG001",
|
|
106
|
+
"ARG002", "PTH108", "SIM108", "PERF401",
|
|
107
|
+
]
|
|
108
|
+
|
|
109
|
+
[tool.ruff.lint.per-file-ignores]
|
|
110
|
+
"__init__.py" = ["F401", "RUF022"]
|
|
111
|
+
"tests/**/*.py" = ["E402", "B905", "S101", "S105", "S106", "S110", "ARG", "TCH", "RUF003", "RUF012", "RUF015", "RUF043", "RUF059"]
|
|
112
|
+
|
|
113
|
+
[tool.mypy]
|
|
114
|
+
python_version = "3.11"
|
|
115
|
+
strict = true
|
|
116
|
+
warn_return_any = true
|
|
117
|
+
warn_unused_configs = true
|
|
118
|
+
|
|
119
|
+
[tool.uv.sources]
|
|
120
|
+
provide-uterm = { workspace = true }
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
#
|
|
2
|
+
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 provide.io llc. All rights reserved.
|
|
3
|
+
# SPDX-License-Identifier: AGPL-3.0-or-later
|
|
4
|
+
#
|
|
5
|
+
|
|
6
|
+
"""Generic swarm manager for agent orchestration.
|
|
7
|
+
|
|
8
|
+
Public API::
|
|
9
|
+
|
|
10
|
+
from provide.uterm.manager import create_manager_app, ManagerConfig, AgentManager
|
|
11
|
+
from provide.uterm.manager.models import AgentStatusBase, SwarmStatus, SpawnBatchRequest
|
|
12
|
+
from provide.uterm.manager.protocols import (
|
|
13
|
+
AccountPoolPlugin, IdentityStorePlugin, ManagedAgentPlugin,
|
|
14
|
+
StatusUpdatePlugin, TimeseriesPlugin, WorkerRegistryPlugin,
|
|
15
|
+
)
|
|
16
|
+
|
|
17
|
+
app, manager = create_manager_app(ManagerConfig(), worker_registry={...})
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
from provide.uterm.manager.app import create_manager_app
|
|
23
|
+
from provide.uterm.manager.config import ManagerConfig
|
|
24
|
+
from provide.uterm.manager.core import AgentManager
|
|
25
|
+
from provide.uterm.manager.models import AgentStatusBase, SwarmStatus
|
|
26
|
+
|
|
27
|
+
__all__ = [
|
|
28
|
+
"AgentManager",
|
|
29
|
+
"AgentStatusBase",
|
|
30
|
+
"ManagerConfig",
|
|
31
|
+
"SwarmStatus",
|
|
32
|
+
"create_manager_app",
|
|
33
|
+
]
|
|
@@ -0,0 +1,271 @@
|
|
|
1
|
+
#
|
|
2
|
+
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 provide.io llc. All rights reserved.
|
|
3
|
+
# SPDX-License-Identifier: AGPL-3.0-or-later
|
|
4
|
+
#
|
|
5
|
+
"""Helper coroutines for AgentProcessManager.monitor_processes.
|
|
6
|
+
|
|
7
|
+
Extracted to keep process.py under the 500-line limit.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import asyncio
|
|
13
|
+
import contextlib
|
|
14
|
+
import time
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
from typing import TYPE_CHECKING, Any
|
|
17
|
+
|
|
18
|
+
from provide.telemetry import get_logger
|
|
19
|
+
|
|
20
|
+
from provide.uterm.manager.constants import ACTIVE_STATES, RUNNING_STATES, TERMINAL_STATES
|
|
21
|
+
from provide.uterm.manager.ext import EVENT_AGENT_EXITED
|
|
22
|
+
|
|
23
|
+
if TYPE_CHECKING:
|
|
24
|
+
import subprocess
|
|
25
|
+
|
|
26
|
+
from provide.uterm.manager.process import AgentProcessManager
|
|
27
|
+
|
|
28
|
+
logger = get_logger(__name__)
|
|
29
|
+
|
|
30
|
+
_STOP_TIMEOUT_S = 5.0
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _set_agent_exit_state(agent: Any, exit_code: int) -> None:
|
|
34
|
+
"""Update agent state fields based on process exit code."""
|
|
35
|
+
if exit_code == 0:
|
|
36
|
+
if agent.state == "error" or agent.error_message:
|
|
37
|
+
agent.state = "error"
|
|
38
|
+
if not agent.exit_reason:
|
|
39
|
+
agent.exit_reason = "reported_error_then_exit_0"
|
|
40
|
+
else:
|
|
41
|
+
agent.state = "completed"
|
|
42
|
+
agent.completed_at = time.time()
|
|
43
|
+
agent.stopped_at = time.time()
|
|
44
|
+
if not agent.exit_reason:
|
|
45
|
+
agent.exit_reason = "target_reached"
|
|
46
|
+
else:
|
|
47
|
+
agent.state = "error"
|
|
48
|
+
if not agent.exit_reason:
|
|
49
|
+
agent.exit_reason = f"exit_code_{exit_code}"
|
|
50
|
+
if not agent.error_message:
|
|
51
|
+
agent.error_message = f"Process exited with code {exit_code}"
|
|
52
|
+
agent.stopped_at = time.time()
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
async def _prune_terminal_agents(
|
|
56
|
+
pm: AgentProcessManager,
|
|
57
|
+
) -> tuple[list[tuple[str, subprocess.Popen[bytes]]], list[Any]]:
|
|
58
|
+
"""Remove terminal-state agents; return (stop_requests, dead_agents)."""
|
|
59
|
+
|
|
60
|
+
prune_stop_requests: list[tuple[str, subprocess.Popen[bytes]]] = []
|
|
61
|
+
async with pm.manager._state_lock:
|
|
62
|
+
dead_agents = [b for b in pm.manager.agents.values() if b.state in TERMINAL_STATES]
|
|
63
|
+
for dead in dead_agents:
|
|
64
|
+
with contextlib.suppress(OSError, RuntimeError):
|
|
65
|
+
pm.release_agent_account(dead.agent_id)
|
|
66
|
+
if (proc := pm.manager.processes.pop(dead.agent_id, None)) is not None:
|
|
67
|
+
prune_stop_requests.append((dead.agent_id, proc))
|
|
68
|
+
pm.manager.agents.pop(dead.agent_id, None)
|
|
69
|
+
return prune_stop_requests, dead_agents
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _collect_spawn_configs(
|
|
73
|
+
active_agents: list[Any],
|
|
74
|
+
dead_agents: list[Any],
|
|
75
|
+
last_config: Any,
|
|
76
|
+
) -> list[Any]:
|
|
77
|
+
"""Collect available spawn configs from active agents, dead agents, or last known config."""
|
|
78
|
+
configs = [b.config for b in active_agents if b.config]
|
|
79
|
+
if not configs:
|
|
80
|
+
configs = [b.config for b in dead_agents if b.config]
|
|
81
|
+
if not configs and last_config:
|
|
82
|
+
configs = [last_config]
|
|
83
|
+
return configs
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _cleanup_old_worker_logs(pm: AgentProcessManager) -> int:
|
|
87
|
+
"""Delete stale .prev and orphan worker log files. Returns count deleted."""
|
|
88
|
+
from provide.uterm.manager.constants import WORKER_LOG_RETENTION_S
|
|
89
|
+
|
|
90
|
+
log_dir = Path(pm._log_dir) if pm._log_dir else Path("logs/workers")
|
|
91
|
+
if not log_dir.is_dir():
|
|
92
|
+
return 0
|
|
93
|
+
cutoff = time.time() - WORKER_LOG_RETENTION_S
|
|
94
|
+
deleted = 0
|
|
95
|
+
active_ids = set(pm.manager.agents.keys())
|
|
96
|
+
for f in log_dir.iterdir():
|
|
97
|
+
if not f.is_file():
|
|
98
|
+
continue
|
|
99
|
+
try:
|
|
100
|
+
mtime = f.stat().st_mtime
|
|
101
|
+
except OSError:
|
|
102
|
+
continue
|
|
103
|
+
if mtime >= cutoff:
|
|
104
|
+
continue
|
|
105
|
+
if f.suffix == ".prev" or (f.name.endswith(".log") and f.stem not in active_ids):
|
|
106
|
+
f.unlink(missing_ok=True)
|
|
107
|
+
deleted += 1
|
|
108
|
+
if deleted:
|
|
109
|
+
logger.info("worker_log_cleanup", deleted=deleted)
|
|
110
|
+
return deleted
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
async def _handle_exited_processes(pm: AgentProcessManager) -> None:
|
|
114
|
+
"""Update state for any agent processes that have exited."""
|
|
115
|
+
async with pm.manager._state_lock:
|
|
116
|
+
exited = [(bid, p) for bid, p in list(pm.manager.processes.items()) if p.poll() is not None]
|
|
117
|
+
for agent_id, process in exited:
|
|
118
|
+
exit_code = process.returncode
|
|
119
|
+
logger.warning(EVENT_AGENT_EXITED, agent_id=agent_id, exit_code=exit_code)
|
|
120
|
+
async with pm.manager._state_lock:
|
|
121
|
+
agent = pm.manager.agents.get(agent_id)
|
|
122
|
+
if agent is None:
|
|
123
|
+
pm.manager.processes.pop(agent_id, None)
|
|
124
|
+
continue
|
|
125
|
+
_set_agent_exit_state(agent, exit_code)
|
|
126
|
+
pm.manager.processes.pop(agent_id, None)
|
|
127
|
+
pm.release_agent_account(agent_id)
|
|
128
|
+
await pm.manager.broadcast_status()
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
async def _handle_heartbeat_timeouts(pm: AgentProcessManager) -> None:
|
|
132
|
+
"""Detect agents that have not sent a heartbeat and mark them as error."""
|
|
133
|
+
now = time.time()
|
|
134
|
+
heartbeat_timeout = pm.manager.config.heartbeat_timeout_s
|
|
135
|
+
heartbeat_timed_out: list[str] = []
|
|
136
|
+
|
|
137
|
+
heartbeat_stop_requests: list[tuple[str, subprocess.Popen[bytes]]] = []
|
|
138
|
+
async with pm.manager._state_lock:
|
|
139
|
+
for agent in list(pm.manager.agents.values()):
|
|
140
|
+
if agent.state in RUNNING_STATES and now - agent.last_update_time > heartbeat_timeout:
|
|
141
|
+
logger.warning("agent_heartbeat_timeout", agent_id=agent.agent_id, timeout_s=heartbeat_timeout)
|
|
142
|
+
agent.state = "error"
|
|
143
|
+
agent.error_message = (
|
|
144
|
+
f"No heartbeat in {heartbeat_timeout:.0f}s - agent process may have crashed or is stuck"
|
|
145
|
+
)
|
|
146
|
+
agent.error_type = "HeartbeatTimeout"
|
|
147
|
+
agent.error_timestamp = time.time()
|
|
148
|
+
agent.exit_reason = "heartbeat_timeout"
|
|
149
|
+
agent.stopped_at = time.time()
|
|
150
|
+
if (proc := pm.manager.processes.pop(agent.agent_id, None)) is not None:
|
|
151
|
+
heartbeat_stop_requests.append((agent.agent_id, proc))
|
|
152
|
+
heartbeat_timed_out.append(agent.agent_id)
|
|
153
|
+
for agent_id, proc in heartbeat_stop_requests:
|
|
154
|
+
await pm._stop_process_tree(agent_id=agent_id, process=proc, timeout_s=_STOP_TIMEOUT_S)
|
|
155
|
+
for bid in heartbeat_timed_out:
|
|
156
|
+
pm.release_agent_account(bid)
|
|
157
|
+
if heartbeat_timed_out:
|
|
158
|
+
await pm.manager.broadcast_status()
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def _handle_stale_queued(pm: AgentProcessManager) -> None:
|
|
162
|
+
"""Launch agents that have been queued too long without spawning."""
|
|
163
|
+
now = time.time()
|
|
164
|
+
for agent in list(pm.manager.agents.values()):
|
|
165
|
+
if agent.state != "queued" or agent.pid != 0 or agent.started_at is not None:
|
|
166
|
+
pm._queued_since.pop(agent.agent_id, None)
|
|
167
|
+
continue
|
|
168
|
+
queued_since = pm._queued_since.get(agent.agent_id)
|
|
169
|
+
if queued_since is None:
|
|
170
|
+
pm._queued_since[agent.agent_id] = now
|
|
171
|
+
continue
|
|
172
|
+
if now - queued_since >= pm._queued_launch_delay:
|
|
173
|
+
if pm.manager.desired_agents > 0:
|
|
174
|
+
pm._queued_since.pop(agent.agent_id, None)
|
|
175
|
+
continue
|
|
176
|
+
logger.warning("stale_queued_agent_launching", agent_id=agent.agent_id, queued_s=round(now - queued_since))
|
|
177
|
+
pm._queued_since.pop(agent.agent_id, None)
|
|
178
|
+
if agent.config:
|
|
179
|
+
task = asyncio.create_task(pm._launch_queued_agent(agent.agent_id, agent.config))
|
|
180
|
+
pm._spawn_tasks.append(task)
|
|
181
|
+
else:
|
|
182
|
+
agent.state = "stopped"
|
|
183
|
+
agent.exit_reason = "no_config"
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
async def _handle_bust_respawn(pm: AgentProcessManager) -> None:
|
|
187
|
+
"""Kill agents that are in bust context if bust_respawn is enabled."""
|
|
188
|
+
if not pm.manager.bust_respawn or pm.manager.swarm_paused:
|
|
189
|
+
return
|
|
190
|
+
now = time.time()
|
|
191
|
+
|
|
192
|
+
bust_stop_requests: list[tuple[str, subprocess.Popen[bytes] | None, int | None]] = []
|
|
193
|
+
for agent in list(pm.manager.agents.values()):
|
|
194
|
+
if agent.state != "running":
|
|
195
|
+
continue
|
|
196
|
+
ctx = str(getattr(agent, "activity_context", "") or "").upper()
|
|
197
|
+
if ctx != "BUST":
|
|
198
|
+
continue
|
|
199
|
+
logger.info("bust_respawn_killing_agent", agent_id=agent.agent_id)
|
|
200
|
+
agent.state = "stopped"
|
|
201
|
+
agent.exit_reason = "bust_respawn"
|
|
202
|
+
agent.stopped_at = now
|
|
203
|
+
proc = pm.manager.processes.pop(agent.agent_id, None)
|
|
204
|
+
pid = agent.pid if agent.pid and agent.pid > 0 else None
|
|
205
|
+
bust_stop_requests.append((agent.agent_id, proc, pid))
|
|
206
|
+
pm.release_agent_account(agent.agent_id)
|
|
207
|
+
for agent_id, proc, pid in bust_stop_requests:
|
|
208
|
+
await pm._stop_process_tree(agent_id=agent_id, process=proc, pid=pid, timeout_s=_STOP_TIMEOUT_S)
|
|
209
|
+
await pm.manager.broadcast_status()
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
async def _spawn_to_desired(
|
|
213
|
+
pm: AgentProcessManager,
|
|
214
|
+
deficit: int,
|
|
215
|
+
active_agents: list[Any],
|
|
216
|
+
dead_agents: list[Any],
|
|
217
|
+
) -> None:
|
|
218
|
+
"""Spawn *deficit* new agents to reach the desired count."""
|
|
219
|
+
configs_available = _collect_spawn_configs(active_agents, dead_agents, pm._last_spawn_config)
|
|
220
|
+
for _ in range(deficit):
|
|
221
|
+
if not configs_available:
|
|
222
|
+
break
|
|
223
|
+
config = configs_available[0]
|
|
224
|
+
async with pm.manager._state_lock:
|
|
225
|
+
new_agent_id = pm.allocate_agent_id()
|
|
226
|
+
if new_agent_id not in pm.manager.agents: # pragma: no branch
|
|
227
|
+
pm.manager.agents[new_agent_id] = pm.manager._agent_status_class(
|
|
228
|
+
agent_id=new_agent_id,
|
|
229
|
+
pid=0,
|
|
230
|
+
config=config,
|
|
231
|
+
state="queued",
|
|
232
|
+
)
|
|
233
|
+
logger.info(
|
|
234
|
+
"desired_state_spawning",
|
|
235
|
+
agent_id=new_agent_id,
|
|
236
|
+
deficit=deficit,
|
|
237
|
+
desired=pm.manager.desired_agents,
|
|
238
|
+
)
|
|
239
|
+
task = asyncio.create_task(pm._launch_queued_agent(new_agent_id, config))
|
|
240
|
+
pm._spawn_tasks.append(task)
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
async def _kill_excess(pm: AgentProcessManager, excess: int, active_agents: list[Any]) -> None:
|
|
244
|
+
"""Kill *excess* agents to trim down to the desired count."""
|
|
245
|
+
to_kill = sorted(active_agents, key=lambda b: b.agent_id, reverse=True)[:excess]
|
|
246
|
+
for agent in to_kill:
|
|
247
|
+
logger.info("desired_state_killing", agent_id=agent.agent_id, excess=excess, desired=pm.manager.desired_agents)
|
|
248
|
+
with contextlib.suppress(OSError, ProcessLookupError, RuntimeError):
|
|
249
|
+
await pm.manager.kill_agent(agent.agent_id)
|
|
250
|
+
with contextlib.suppress(OSError, RuntimeError):
|
|
251
|
+
pm.release_agent_account(agent.agent_id)
|
|
252
|
+
async with pm.manager._state_lock:
|
|
253
|
+
pm.manager.agents.pop(agent.agent_id, None)
|
|
254
|
+
pm.manager.processes.pop(agent.agent_id, None)
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
async def _handle_desired_state(pm: AgentProcessManager) -> None:
|
|
258
|
+
"""Enforce the desired agent count: spawn deficits, kill excesses."""
|
|
259
|
+
if pm.manager.desired_agents <= 0 or pm.manager.swarm_paused:
|
|
260
|
+
return
|
|
261
|
+
prune_stop_requests, dead_agents = await _prune_terminal_agents(pm)
|
|
262
|
+
async with pm.manager._state_lock:
|
|
263
|
+
active_agents = [b for b in pm.manager.agents.values() if b.state in ACTIVE_STATES]
|
|
264
|
+
deficit = pm.manager.desired_agents - len(active_agents)
|
|
265
|
+
|
|
266
|
+
if deficit > 0:
|
|
267
|
+
await _spawn_to_desired(pm, deficit, active_agents, dead_agents)
|
|
268
|
+
elif deficit < 0:
|
|
269
|
+
await _kill_excess(pm, -deficit, active_agents)
|
|
270
|
+
for agent_id, proc in prune_stop_requests:
|
|
271
|
+
await pm._stop_process_tree(agent_id=agent_id, process=proc, timeout_s=_STOP_TIMEOUT_S)
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
#
|
|
2
|
+
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 provide.io llc. All rights reserved.
|
|
3
|
+
# SPDX-License-Identifier: AGPL-3.0-or-later
|
|
4
|
+
#
|
|
5
|
+
"""Application factory for the generic swarm manager."""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import os
|
|
10
|
+
from typing import TYPE_CHECKING, Any
|
|
11
|
+
|
|
12
|
+
from fastapi import APIRouter, FastAPI, WebSocket, WebSocketDisconnect
|
|
13
|
+
from fastapi.middleware.cors import CORSMiddleware
|
|
14
|
+
from provide.telemetry import get_logger
|
|
15
|
+
|
|
16
|
+
from provide.uterm.manager.auth import setup_auth
|
|
17
|
+
from provide.uterm.manager.core import AgentManager
|
|
18
|
+
from provide.uterm.manager.process import AgentProcessManager
|
|
19
|
+
from provide.uterm.manager.routes import router as swarm_router
|
|
20
|
+
|
|
21
|
+
if TYPE_CHECKING:
|
|
22
|
+
from provide.uterm.manager.config import ManagerConfig
|
|
23
|
+
from provide.uterm.manager.models import AgentStatusBase
|
|
24
|
+
from provide.uterm.manager.protocols import (
|
|
25
|
+
AccountPoolPlugin,
|
|
26
|
+
IdentityStorePlugin,
|
|
27
|
+
ManagedAgentPlugin,
|
|
28
|
+
StatusUpdatePlugin,
|
|
29
|
+
TimeseriesPlugin,
|
|
30
|
+
WorkerRegistryPlugin,
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
logger = get_logger(__name__)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def create_manager_app(
|
|
37
|
+
config: ManagerConfig,
|
|
38
|
+
*,
|
|
39
|
+
agent_status_class: type[AgentStatusBase] | None = None,
|
|
40
|
+
worker_registry: dict[str, WorkerRegistryPlugin] | None = None,
|
|
41
|
+
account_pool: AccountPoolPlugin | None = None,
|
|
42
|
+
identity_store: IdentityStorePlugin | None = None,
|
|
43
|
+
managed_agent: ManagedAgentPlugin | None = None,
|
|
44
|
+
status_update: StatusUpdatePlugin | None = None,
|
|
45
|
+
timeseries: TimeseriesPlugin | None = None,
|
|
46
|
+
swarm_status_builder: Any | None = None,
|
|
47
|
+
extra_routers: list[APIRouter] | None = None,
|
|
48
|
+
) -> tuple[FastAPI, AgentManager]:
|
|
49
|
+
"""Create a FastAPI application wired to a generic AgentManager.
|
|
50
|
+
|
|
51
|
+
Returns ``(app, manager)`` so the caller can further customise
|
|
52
|
+
either before starting the server.
|
|
53
|
+
"""
|
|
54
|
+
manager = AgentManager(
|
|
55
|
+
config,
|
|
56
|
+
agent_status_class=agent_status_class,
|
|
57
|
+
account_pool=account_pool,
|
|
58
|
+
identity_store=identity_store,
|
|
59
|
+
status_update=status_update,
|
|
60
|
+
timeseries_plugin=timeseries,
|
|
61
|
+
swarm_status_builder=swarm_status_builder,
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
process_mgr = AgentProcessManager(
|
|
65
|
+
manager,
|
|
66
|
+
worker_registry=worker_registry,
|
|
67
|
+
log_dir=config.log_dir,
|
|
68
|
+
)
|
|
69
|
+
if config.spawn_policy_webhook_url:
|
|
70
|
+
from provide.uterm.manager.ext import WebhookAgentSpawnPolicyGate
|
|
71
|
+
|
|
72
|
+
process_mgr.set_policy_gate(
|
|
73
|
+
WebhookAgentSpawnPolicyGate(
|
|
74
|
+
url=config.spawn_policy_webhook_url,
|
|
75
|
+
secret=config.spawn_policy_webhook_secret,
|
|
76
|
+
timeout_s=config.spawn_policy_webhook_timeout_s,
|
|
77
|
+
)
|
|
78
|
+
)
|
|
79
|
+
manager.agent_process_manager = process_mgr
|
|
80
|
+
|
|
81
|
+
app = FastAPI(title=config.title)
|
|
82
|
+
manager.app = app
|
|
83
|
+
|
|
84
|
+
# Auth
|
|
85
|
+
setup_auth(app, env_var=config.auth_token_env_var, config=config)
|
|
86
|
+
|
|
87
|
+
# CORS — credentials are always allowed, so a wildcard or empty origin list
|
|
88
|
+
# would expose this process-kill/spawn API to credentialed cross-site
|
|
89
|
+
# requests. Refuse such a config at build time and require an explicit
|
|
90
|
+
# allowlist.
|
|
91
|
+
cors_env = os.environ.get("UTERM_CORS_ORIGINS", "").strip()
|
|
92
|
+
origins = [o.strip() for o in cors_env.split(",") if o.strip()] if cors_env else config.cors_origins
|
|
93
|
+
if not origins:
|
|
94
|
+
raise ValueError(
|
|
95
|
+
"CORS origin allowlist is empty; credentialed CORS requires an explicit list of origins "
|
|
96
|
+
"(set UTERM_CORS_ORIGINS or ManagerConfig.cors_origins)"
|
|
97
|
+
)
|
|
98
|
+
if "*" in origins:
|
|
99
|
+
raise ValueError(
|
|
100
|
+
"CORS wildcard origin '*' is not allowed when credentials are enabled; list explicit origins instead"
|
|
101
|
+
)
|
|
102
|
+
app.add_middleware(
|
|
103
|
+
CORSMiddleware,
|
|
104
|
+
allow_origins=origins,
|
|
105
|
+
allow_credentials=True,
|
|
106
|
+
allow_methods=["GET", "POST", "PUT", "DELETE", "OPTIONS"],
|
|
107
|
+
allow_headers=["*"],
|
|
108
|
+
)
|
|
109
|
+
|
|
110
|
+
# Wire state
|
|
111
|
+
app.state.swarm_manager = manager
|
|
112
|
+
if managed_agent is not None:
|
|
113
|
+
app.state.managed_agent_plugin = managed_agent
|
|
114
|
+
|
|
115
|
+
# Include routes
|
|
116
|
+
app.include_router(swarm_router)
|
|
117
|
+
for extra in extra_routers or []:
|
|
118
|
+
app.include_router(extra)
|
|
119
|
+
|
|
120
|
+
# WebSocket endpoint for dashboard push updates
|
|
121
|
+
@app.websocket("/ws/swarm")
|
|
122
|
+
async def websocket_endpoint(websocket: WebSocket) -> None:
|
|
123
|
+
await websocket.accept()
|
|
124
|
+
async with manager._ws_lock:
|
|
125
|
+
manager.websocket_clients.add(websocket)
|
|
126
|
+
try:
|
|
127
|
+
while True:
|
|
128
|
+
await websocket.receive_text()
|
|
129
|
+
except WebSocketDisconnect:
|
|
130
|
+
async with manager._ws_lock:
|
|
131
|
+
manager.websocket_clients.discard(websocket)
|
|
132
|
+
except Exception as e: # pragma: no cover — requires mid-stream WS failure
|
|
133
|
+
logger.exception("websocket_error", error=str(e))
|
|
134
|
+
async with manager._ws_lock:
|
|
135
|
+
manager.websocket_clients.discard(websocket)
|
|
136
|
+
|
|
137
|
+
# WebSocket endpoint for MCP client lifecycle tracking
|
|
138
|
+
@app.websocket("/ws/mcp-client")
|
|
139
|
+
async def mcp_client_endpoint(websocket: WebSocket) -> None:
|
|
140
|
+
await websocket.accept()
|
|
141
|
+
await manager.register_mcp_client(websocket)
|
|
142
|
+
try:
|
|
143
|
+
while True:
|
|
144
|
+
await websocket.receive_text()
|
|
145
|
+
except WebSocketDisconnect:
|
|
146
|
+
await manager.unregister_mcp_client(websocket)
|
|
147
|
+
except Exception as e: # pragma: no cover - requires mid-stream WS failure
|
|
148
|
+
logger.exception("mcp_client_websocket_error", error=str(e))
|
|
149
|
+
await manager.unregister_mcp_client(websocket)
|
|
150
|
+
|
|
151
|
+
# Load persisted state
|
|
152
|
+
manager._load_state()
|
|
153
|
+
|
|
154
|
+
return app, manager
|