sinapsis-api 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sinapsis_api/__init__.py +1 -0
- sinapsis_api/agent_router.py +482 -0
- sinapsis_api/agent_worker.py +240 -0
- sinapsis_api/api/__init__.py +1 -0
- sinapsis_api/api/healthcheck.py +25 -0
- sinapsis_api/api/main.py +40 -0
- sinapsis_api/api/routes/_common.py +85 -0
- sinapsis_api/api/routes/agents.py +91 -0
- sinapsis_api/api/routes/health.py +18 -0
- sinapsis_api/api/routes/jobs.py +81 -0
- sinapsis_api/client.py +135 -0
- sinapsis_api/constants.py +57 -0
- sinapsis_api/docker_runtime/__init__.py +42 -0
- sinapsis_api/docker_runtime/_file.py +71 -0
- sinapsis_api/docker_runtime/base.py +290 -0
- sinapsis_api/docker_runtime/image.py +83 -0
- sinapsis_api/docker_runtime/runtime.py +143 -0
- sinapsis_api/docker_runtime/services.py +127 -0
- sinapsis_api/env_vars/__init__.py +19 -0
- sinapsis_api/env_vars/env_vars.py +202 -0
- sinapsis_api/journal.py +298 -0
- sinapsis_api/models.py +30 -0
- sinapsis_api/onexth_runtime.py +68 -0
- sinapsis_api/protocol.py +106 -0
- sinapsis_api/telemetry.py +186 -0
- sinapsis_api-1.0.0.dist-info/METADATA +280 -0
- sinapsis_api-1.0.0.dist-info/RECORD +30 -0
- sinapsis_api-1.0.0.dist-info/WHEEL +4 -0
- sinapsis_api-1.0.0.dist-info/licenses/LICENSE-APACHE +202 -0
- sinapsis_api-1.0.0.dist-info/licenses/LICENSE-MIT +21 -0
sinapsis_api/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Sinapsis local multi-process agent API."""
|
|
@@ -0,0 +1,482 @@
|
|
|
1
|
+
"""Single Router process for agent lifecycle, commands, journals, and streams."""
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
import sys
|
|
5
|
+
import uuid
|
|
6
|
+
from collections import OrderedDict
|
|
7
|
+
from dataclasses import dataclass
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
import zmq
|
|
11
|
+
import zmq.asyncio
|
|
12
|
+
from sinapsis.logging import sinapsis_logger
|
|
13
|
+
|
|
14
|
+
from sinapsis_api.constants import (
|
|
15
|
+
ControlType,
|
|
16
|
+
JobStatus,
|
|
17
|
+
RouterError,
|
|
18
|
+
StreamFrame,
|
|
19
|
+
StreamOperation,
|
|
20
|
+
WorkerFrame,
|
|
21
|
+
)
|
|
22
|
+
from sinapsis_api.env_vars import (
|
|
23
|
+
SINAPSIS_API_AGENT_START_TIMEOUT_SECONDS,
|
|
24
|
+
SINAPSIS_API_COMMAND_CACHE_SIZE,
|
|
25
|
+
SINAPSIS_API_EVENT_MAX_BYTES,
|
|
26
|
+
SINAPSIS_API_EVENT_MAX_BYTES_PER_AGENT,
|
|
27
|
+
SINAPSIS_API_EVENT_MAX_EVENTS_PER_AGENT,
|
|
28
|
+
SINAPSIS_API_EVENT_REPLAY_LIMIT,
|
|
29
|
+
SINAPSIS_API_EVENT_REPLAY_MAX_BYTES,
|
|
30
|
+
SINAPSIS_API_EVENT_RETENTION_SECONDS,
|
|
31
|
+
SINAPSIS_API_JOURNAL_GC_INTERVAL_SECONDS,
|
|
32
|
+
SINAPSIS_API_ROUTER_AGENT_ENDPOINT,
|
|
33
|
+
SINAPSIS_API_ROUTER_API_ENDPOINT,
|
|
34
|
+
SINAPSIS_API_ROUTER_STREAM_ENDPOINT,
|
|
35
|
+
SINAPSIS_API_STOPPED_AGENT_RETENTION_SECONDS,
|
|
36
|
+
SINAPSIS_API_SUBSCRIBER_QUEUE_SIZE,
|
|
37
|
+
SINAPSIS_API_ZMQ_RECEIVE_HWM,
|
|
38
|
+
SINAPSIS_API_ZMQ_SEND_HWM,
|
|
39
|
+
)
|
|
40
|
+
from sinapsis_api.journal import AgentEventJournals, ReplayResult
|
|
41
|
+
from sinapsis_api.models import AgentHandle, JobRecord
|
|
42
|
+
from sinapsis_api.protocol import EventRecord, pack_control, unpack_control
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
@dataclass(frozen=True, slots=True)
|
|
46
|
+
class RouterConfig:
|
|
47
|
+
"""Router configuration sourced from environment-backed constants."""
|
|
48
|
+
|
|
49
|
+
api_endpoint: str = SINAPSIS_API_ROUTER_API_ENDPOINT
|
|
50
|
+
agent_endpoint: str = SINAPSIS_API_ROUTER_AGENT_ENDPOINT
|
|
51
|
+
stream_endpoint: str = SINAPSIS_API_ROUTER_STREAM_ENDPOINT
|
|
52
|
+
retention_seconds: float = SINAPSIS_API_EVENT_RETENTION_SECONDS
|
|
53
|
+
max_events_per_agent: int = SINAPSIS_API_EVENT_MAX_EVENTS_PER_AGENT
|
|
54
|
+
max_bytes_per_agent: int = SINAPSIS_API_EVENT_MAX_BYTES_PER_AGENT
|
|
55
|
+
max_event_bytes: int = SINAPSIS_API_EVENT_MAX_BYTES
|
|
56
|
+
replay_limit: int = SINAPSIS_API_EVENT_REPLAY_LIMIT
|
|
57
|
+
replay_max_bytes: int = SINAPSIS_API_EVENT_REPLAY_MAX_BYTES
|
|
58
|
+
subscriber_queue_size: int = SINAPSIS_API_SUBSCRIBER_QUEUE_SIZE
|
|
59
|
+
agent_start_timeout_seconds: float = SINAPSIS_API_AGENT_START_TIMEOUT_SECONDS
|
|
60
|
+
stopped_agent_retention_seconds: float = SINAPSIS_API_STOPPED_AGENT_RETENTION_SECONDS
|
|
61
|
+
journal_gc_interval_seconds: float = SINAPSIS_API_JOURNAL_GC_INTERVAL_SECONDS
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
class SinapsisAgentRouter:
|
|
65
|
+
"""Own all agent processes and stateless per-agent event streams."""
|
|
66
|
+
|
|
67
|
+
def __init__(self, config: RouterConfig | None = None) -> None:
|
|
68
|
+
self.config = config or RouterConfig()
|
|
69
|
+
self.stream_id = str(uuid.uuid4())
|
|
70
|
+
self.context = zmq.asyncio.Context.instance()
|
|
71
|
+
self.journals = AgentEventJournals(
|
|
72
|
+
stream_id=self.stream_id,
|
|
73
|
+
retention_seconds=self.config.retention_seconds,
|
|
74
|
+
max_events=self.config.max_events_per_agent,
|
|
75
|
+
max_bytes=self.config.max_bytes_per_agent,
|
|
76
|
+
max_event_bytes=self.config.max_event_bytes,
|
|
77
|
+
stopped_agent_retention_seconds=self.config.stopped_agent_retention_seconds,
|
|
78
|
+
)
|
|
79
|
+
self.agents: dict[str, AgentHandle] = {}
|
|
80
|
+
self.agent_by_identity: dict[bytes, AgentHandle] = {}
|
|
81
|
+
self.jobs: dict[str, JobRecord] = {}
|
|
82
|
+
self.command_results: OrderedDict[str, dict[str, Any]] = OrderedDict()
|
|
83
|
+
self._background_tasks: set[asyncio.Task[Any]] = set()
|
|
84
|
+
|
|
85
|
+
async def run(self) -> None:
|
|
86
|
+
"""Bind IPC sockets and serve until cancelled."""
|
|
87
|
+
api_socket = self._router_socket(self.config.api_endpoint)
|
|
88
|
+
agent_socket = self._router_socket(self.config.agent_endpoint)
|
|
89
|
+
stream_socket = self._router_socket(self.config.stream_endpoint)
|
|
90
|
+
sinapsis_logger.info("Router started with stream_id=%s.", self.stream_id)
|
|
91
|
+
try:
|
|
92
|
+
async with asyncio.TaskGroup() as task_group:
|
|
93
|
+
task_group.create_task(self._api_loop(api_socket, agent_socket))
|
|
94
|
+
task_group.create_task(self._agent_loop(agent_socket))
|
|
95
|
+
task_group.create_task(self._stream_loop(stream_socket))
|
|
96
|
+
task_group.create_task(self._process_watch_loop())
|
|
97
|
+
task_group.create_task(self._journal_gc_loop())
|
|
98
|
+
except Exception:
|
|
99
|
+
sinapsis_logger.exception("Router stopped because of an unhandled exception.")
|
|
100
|
+
raise
|
|
101
|
+
finally:
|
|
102
|
+
await self._terminate_agents()
|
|
103
|
+
api_socket.close()
|
|
104
|
+
agent_socket.close()
|
|
105
|
+
stream_socket.close()
|
|
106
|
+
|
|
107
|
+
def _router_socket(self, endpoint: str) -> zmq.asyncio.Socket:
|
|
108
|
+
socket = self.context.socket(zmq.ROUTER)
|
|
109
|
+
socket.setsockopt(zmq.LINGER, 0)
|
|
110
|
+
socket.setsockopt(zmq.SNDHWM, SINAPSIS_API_ZMQ_SEND_HWM)
|
|
111
|
+
socket.setsockopt(zmq.RCVHWM, SINAPSIS_API_ZMQ_RECEIVE_HWM)
|
|
112
|
+
socket.bind(endpoint)
|
|
113
|
+
return socket
|
|
114
|
+
|
|
115
|
+
async def _api_loop(self, api_socket: zmq.asyncio.Socket, agent_socket: zmq.asyncio.Socket) -> None:
|
|
116
|
+
while True:
|
|
117
|
+
frames = await api_socket.recv_multipart(copy=False)
|
|
118
|
+
identity = bytes(frames[0])
|
|
119
|
+
try:
|
|
120
|
+
response = await self._handle_api_request(unpack_control(frames[-1].buffer), agent_socket)
|
|
121
|
+
except KeyError as exception:
|
|
122
|
+
sinapsis_logger.warning("Router request referenced missing state: %s", exception)
|
|
123
|
+
response = self._error(RouterError.NOT_FOUND, str(exception))
|
|
124
|
+
except ValueError as exception:
|
|
125
|
+
sinapsis_logger.warning("Router rejected request: %s", exception)
|
|
126
|
+
response = self._error(RouterError.CONFLICT, str(exception))
|
|
127
|
+
except Exception as exception:
|
|
128
|
+
sinapsis_logger.exception("Router API request failed.")
|
|
129
|
+
response = self._error(RouterError.INTERNAL, str(exception))
|
|
130
|
+
await api_socket.send_multipart([identity, pack_control(response)])
|
|
131
|
+
|
|
132
|
+
async def _handle_api_request(self, request: dict[str, Any], agent_socket: zmq.asyncio.Socket) -> dict[str, Any]:
|
|
133
|
+
command_id = request.get("command_id")
|
|
134
|
+
if command_id is not None and command_id in self.command_results:
|
|
135
|
+
return self.command_results[command_id]
|
|
136
|
+
|
|
137
|
+
control_type = ControlType(request["type"])
|
|
138
|
+
if control_type == ControlType.HEALTH:
|
|
139
|
+
result = {"ok": True, "status": "ready", "stream_id": self.stream_id}
|
|
140
|
+
elif control_type == ControlType.LIST_AGENTS:
|
|
141
|
+
result = {"ok": True, "agents": list(self.agents), "stream_id": self.stream_id}
|
|
142
|
+
elif control_type == ControlType.START_AGENT:
|
|
143
|
+
await self.start_agent(request["agent_id"], request["agent_definition"], agent_socket)
|
|
144
|
+
result = {"ok": True, "stream_id": self.stream_id}
|
|
145
|
+
elif control_type == ControlType.STOP_AGENT:
|
|
146
|
+
await self.stop_agent(request["agent_id"], agent_socket)
|
|
147
|
+
result = {"ok": True}
|
|
148
|
+
elif control_type == ControlType.START_JOB:
|
|
149
|
+
result = await self._start_job(request, agent_socket)
|
|
150
|
+
elif control_type in {ControlType.CANCEL_JOB, ControlType.APPROVAL_RESPONSE}:
|
|
151
|
+
await self._route_job_command(request, agent_socket)
|
|
152
|
+
result = {"ok": True}
|
|
153
|
+
elif control_type == ControlType.GET_JOB:
|
|
154
|
+
job = self._require_job(request["job_id"])
|
|
155
|
+
result = {"ok": True, "job": self._job_dict(job)}
|
|
156
|
+
elif control_type == ControlType.EVENT_BOUNDS:
|
|
157
|
+
window = await self.journals.bounds(request["agent_id"])
|
|
158
|
+
result = {"ok": True, "window": self._window_dict(window)}
|
|
159
|
+
else:
|
|
160
|
+
sinapsis_logger.error("Unsupported control type: %s.", control_type.value)
|
|
161
|
+
raise ValueError(f"Unsupported control type: {control_type.value}.")
|
|
162
|
+
|
|
163
|
+
if command_id is not None:
|
|
164
|
+
self._cache_command_result(command_id, result)
|
|
165
|
+
return result
|
|
166
|
+
|
|
167
|
+
async def _start_job(self, request: dict[str, Any], agent_socket: zmq.asyncio.Socket) -> dict[str, Any]:
|
|
168
|
+
job_id = request["job_id"]
|
|
169
|
+
agent_id = request["agent_id"]
|
|
170
|
+
handle = self.agents.get(agent_id)
|
|
171
|
+
if handle is None or not handle.ready.is_set():
|
|
172
|
+
sinapsis_logger.error("Unknown or unready agent: %s.", agent_id)
|
|
173
|
+
raise KeyError(f"Unknown or unready agent: {agent_id}.")
|
|
174
|
+
if job_id in self.jobs:
|
|
175
|
+
sinapsis_logger.error("Job already exists: %s.", job_id)
|
|
176
|
+
raise ValueError(f"Job already exists: {job_id}.")
|
|
177
|
+
self.jobs[job_id] = JobRecord(job_id, agent_id, handle.instance_id)
|
|
178
|
+
await agent_socket.send_multipart([handle.identity, WorkerFrame.COMMAND.value, pack_control(request)])
|
|
179
|
+
return {"ok": True, "job_id": job_id, "agent_id": agent_id}
|
|
180
|
+
|
|
181
|
+
async def _route_job_command(self, request: dict[str, Any], agent_socket: zmq.asyncio.Socket) -> None:
|
|
182
|
+
job = self._require_job(request["job_id"])
|
|
183
|
+
handle = self.agents.get(job.agent_id)
|
|
184
|
+
if handle is None:
|
|
185
|
+
sinapsis_logger.error("Agent is not running: %s.", job.agent_id)
|
|
186
|
+
raise RuntimeError(f"Agent is not running: {job.agent_id}.")
|
|
187
|
+
await agent_socket.send_multipart([handle.identity, WorkerFrame.COMMAND.value, pack_control(request)])
|
|
188
|
+
|
|
189
|
+
async def _agent_loop(self, agent_socket: zmq.asyncio.Socket) -> None:
|
|
190
|
+
while True:
|
|
191
|
+
frames = await agent_socket.recv_multipart(copy=False)
|
|
192
|
+
identity = bytes(frames[0])
|
|
193
|
+
frame_type = bytes(frames[1])
|
|
194
|
+
if frame_type == WorkerFrame.HELLO.value:
|
|
195
|
+
await self._register_worker(identity, unpack_control(frames[2].buffer), agent_socket)
|
|
196
|
+
continue
|
|
197
|
+
handle = self.agent_by_identity.get(identity)
|
|
198
|
+
if handle is None:
|
|
199
|
+
sinapsis_logger.warning("Ignoring frame from unknown worker identity.")
|
|
200
|
+
continue
|
|
201
|
+
if frame_type == WorkerFrame.READY.value:
|
|
202
|
+
handle.ready.set()
|
|
203
|
+
elif frame_type == WorkerFrame.EVENT.value:
|
|
204
|
+
job_id_bytes = bytes(frames[2])
|
|
205
|
+
try:
|
|
206
|
+
await self.journals.append(
|
|
207
|
+
agent_id=handle.agent_id,
|
|
208
|
+
instance_id=handle.instance_id,
|
|
209
|
+
job_id=job_id_bytes.decode() if job_id_bytes else None,
|
|
210
|
+
payload=bytes(frames[3].buffer),
|
|
211
|
+
)
|
|
212
|
+
except ValueError:
|
|
213
|
+
sinapsis_logger.exception("Dropping oversized event from agent %s.", handle.agent_id)
|
|
214
|
+
elif frame_type == WorkerFrame.JOB_STATE.value:
|
|
215
|
+
state = unpack_control(frames[2].buffer)
|
|
216
|
+
job = self.jobs.get(state["job_id"])
|
|
217
|
+
if job is not None:
|
|
218
|
+
job.status = state["status"]
|
|
219
|
+
job.error = state.get("error")
|
|
220
|
+
|
|
221
|
+
async def _register_worker(self, identity: bytes, hello: dict[str, Any], agent_socket: zmq.asyncio.Socket) -> None:
|
|
222
|
+
handle = self.agents.get(hello["agent_id"])
|
|
223
|
+
if handle is None or handle.instance_id != hello["instance_id"]:
|
|
224
|
+
sinapsis_logger.warning("Ignoring stale worker registration for agent %s.", hello["agent_id"])
|
|
225
|
+
return
|
|
226
|
+
handle.identity = identity
|
|
227
|
+
self.agent_by_identity[identity] = handle
|
|
228
|
+
await agent_socket.send_multipart(
|
|
229
|
+
[
|
|
230
|
+
identity,
|
|
231
|
+
WorkerFrame.COMMAND.value,
|
|
232
|
+
pack_control({"type": ControlType.INIT.value, "agent_definition": handle.agent_definition}),
|
|
233
|
+
]
|
|
234
|
+
)
|
|
235
|
+
|
|
236
|
+
async def _stream_loop(self, stream_socket: zmq.asyncio.Socket) -> None:
|
|
237
|
+
while True:
|
|
238
|
+
frames = await stream_socket.recv_multipart(copy=False)
|
|
239
|
+
identity = bytes(frames[0])
|
|
240
|
+
operation = StreamOperation(bytes(frames[1]))
|
|
241
|
+
parameters = unpack_control(frames[2].buffer)
|
|
242
|
+
if operation == StreamOperation.REPLAY:
|
|
243
|
+
self._track(asyncio.create_task(self._serve_replay(stream_socket, identity, parameters)))
|
|
244
|
+
elif operation == StreamOperation.SUBSCRIBE:
|
|
245
|
+
self._track(asyncio.create_task(self._serve_subscription(stream_socket, identity, parameters)))
|
|
246
|
+
|
|
247
|
+
async def _serve_replay(self, socket: zmq.asyncio.Socket, identity: bytes, parameters: dict[str, Any]) -> None:
|
|
248
|
+
try:
|
|
249
|
+
replay = await self.journals.replay(
|
|
250
|
+
agent_id=parameters["agent_id"],
|
|
251
|
+
after=int(parameters.get("after", 0)),
|
|
252
|
+
job_id=parameters.get("job_id"),
|
|
253
|
+
limit=min(int(parameters.get("limit", self.config.replay_limit)), self.config.replay_limit),
|
|
254
|
+
max_bytes=self.config.replay_max_bytes,
|
|
255
|
+
)
|
|
256
|
+
for record in replay.events:
|
|
257
|
+
await self._send_event(socket, identity, record)
|
|
258
|
+
await socket.send_multipart([identity, StreamFrame.DONE.value, pack_control(self._replay_dict(replay))])
|
|
259
|
+
except Exception as exception:
|
|
260
|
+
sinapsis_logger.exception("Event replay failed.")
|
|
261
|
+
await socket.send_multipart(
|
|
262
|
+
[
|
|
263
|
+
identity,
|
|
264
|
+
StreamFrame.DONE.value,
|
|
265
|
+
pack_control(self._error(RouterError.NOT_FOUND, str(exception))),
|
|
266
|
+
]
|
|
267
|
+
)
|
|
268
|
+
|
|
269
|
+
async def _serve_subscription(
|
|
270
|
+
self, socket: zmq.asyncio.Socket, identity: bytes, parameters: dict[str, Any]
|
|
271
|
+
) -> None:
|
|
272
|
+
agent_id = parameters["agent_id"]
|
|
273
|
+
subscription_id: int | None = None
|
|
274
|
+
try:
|
|
275
|
+
subscription_id, replay, subscription = await self.journals.subscribe(
|
|
276
|
+
agent_id=agent_id,
|
|
277
|
+
after=parameters.get("after"),
|
|
278
|
+
job_id=parameters.get("job_id"),
|
|
279
|
+
queue_size=self.config.subscriber_queue_size,
|
|
280
|
+
replay_limit=self.config.replay_limit,
|
|
281
|
+
replay_max_bytes=self.config.replay_max_bytes,
|
|
282
|
+
)
|
|
283
|
+
if replay is not None:
|
|
284
|
+
replay_boundary = subscription.after
|
|
285
|
+
while True:
|
|
286
|
+
for record in replay.events:
|
|
287
|
+
await self._send_event(socket, identity, record)
|
|
288
|
+
if not replay.has_more:
|
|
289
|
+
break
|
|
290
|
+
replay = await self.journals.replay_through(
|
|
291
|
+
agent_id=agent_id,
|
|
292
|
+
after=replay.next_cursor,
|
|
293
|
+
through=replay_boundary,
|
|
294
|
+
job_id=parameters.get("job_id"),
|
|
295
|
+
limit=self.config.replay_limit,
|
|
296
|
+
max_bytes=self.config.replay_max_bytes,
|
|
297
|
+
)
|
|
298
|
+
await socket.send_multipart(
|
|
299
|
+
[
|
|
300
|
+
identity,
|
|
301
|
+
StreamFrame.LIVE.value,
|
|
302
|
+
pack_control({"stream_id": self.stream_id, "after": subscription.after}),
|
|
303
|
+
]
|
|
304
|
+
)
|
|
305
|
+
while not subscription.closed:
|
|
306
|
+
record = await subscription.queue.get()
|
|
307
|
+
await self._send_event(socket, identity, record)
|
|
308
|
+
await socket.send_multipart(
|
|
309
|
+
[
|
|
310
|
+
identity,
|
|
311
|
+
StreamFrame.LAGGED.value,
|
|
312
|
+
pack_control({"stream_id": self.stream_id, "resume_after": subscription.after}),
|
|
313
|
+
]
|
|
314
|
+
)
|
|
315
|
+
except Exception as exception:
|
|
316
|
+
sinapsis_logger.exception("Live subscription failed for agent %s.", agent_id)
|
|
317
|
+
await socket.send_multipart(
|
|
318
|
+
[
|
|
319
|
+
identity,
|
|
320
|
+
StreamFrame.LAGGED.value,
|
|
321
|
+
pack_control(self._error(RouterError.NOT_FOUND, str(exception))),
|
|
322
|
+
]
|
|
323
|
+
)
|
|
324
|
+
finally:
|
|
325
|
+
if subscription_id is not None:
|
|
326
|
+
await self.journals.unsubscribe(agent_id, subscription_id)
|
|
327
|
+
|
|
328
|
+
async def _send_event(self, socket: zmq.asyncio.Socket, identity: bytes, record: EventRecord) -> None:
|
|
329
|
+
metadata, payload = record.zmq_frames()
|
|
330
|
+
await socket.send_multipart([identity, StreamFrame.EVENT.value, metadata, payload], copy=False)
|
|
331
|
+
|
|
332
|
+
async def start_agent(
|
|
333
|
+
self, agent_id: str, agent_definition: dict[str, Any], agent_socket: zmq.asyncio.Socket
|
|
334
|
+
) -> None:
|
|
335
|
+
"""Start one isolated worker and preserve any existing logical-agent journal."""
|
|
336
|
+
if agent_id in self.agents:
|
|
337
|
+
sinapsis_logger.error("Agent already running: %s.", agent_id)
|
|
338
|
+
raise ValueError(f"Agent already running: {agent_id}.")
|
|
339
|
+
await self.journals.ensure_agent(agent_id)
|
|
340
|
+
instance_id = str(uuid.uuid4())
|
|
341
|
+
identity = f"agent:{agent_id}:{instance_id}".encode()
|
|
342
|
+
process = await asyncio.create_subprocess_exec(
|
|
343
|
+
sys.executable,
|
|
344
|
+
"-m",
|
|
345
|
+
"sinapsis_api.agent_worker",
|
|
346
|
+
"--endpoint",
|
|
347
|
+
self.config.agent_endpoint,
|
|
348
|
+
"--agent-id",
|
|
349
|
+
agent_id,
|
|
350
|
+
"--instance-id",
|
|
351
|
+
instance_id,
|
|
352
|
+
)
|
|
353
|
+
handle = AgentHandle(agent_id, instance_id, process, identity, agent_definition)
|
|
354
|
+
self.agents[agent_id] = handle
|
|
355
|
+
self.agent_by_identity[identity] = handle
|
|
356
|
+
try:
|
|
357
|
+
await asyncio.wait_for(handle.ready.wait(), self.config.agent_start_timeout_seconds)
|
|
358
|
+
except Exception:
|
|
359
|
+
sinapsis_logger.exception("Agent %s failed to become ready.", agent_id)
|
|
360
|
+
self.agents.pop(agent_id, None)
|
|
361
|
+
self.agent_by_identity.pop(identity, None)
|
|
362
|
+
await self.journals.mark_stopped(agent_id)
|
|
363
|
+
if process.returncode is None:
|
|
364
|
+
process.terminate()
|
|
365
|
+
raise
|
|
366
|
+
|
|
367
|
+
async def stop_agent(self, agent_id: str, agent_socket: zmq.asyncio.Socket) -> None:
|
|
368
|
+
"""Request worker shutdown without deleting retained event history."""
|
|
369
|
+
handle = self.agents.get(agent_id)
|
|
370
|
+
if handle is None:
|
|
371
|
+
sinapsis_logger.error("Unknown running agent: %s.", agent_id)
|
|
372
|
+
raise KeyError(f"Unknown running agent: {agent_id}.")
|
|
373
|
+
await agent_socket.send_multipart(
|
|
374
|
+
[
|
|
375
|
+
handle.identity,
|
|
376
|
+
WorkerFrame.COMMAND.value,
|
|
377
|
+
pack_control({"type": ControlType.STOP.value}),
|
|
378
|
+
]
|
|
379
|
+
)
|
|
380
|
+
|
|
381
|
+
async def _process_watch_loop(self) -> None:
|
|
382
|
+
while True:
|
|
383
|
+
await asyncio.sleep(0.25)
|
|
384
|
+
for agent_id, handle in tuple(self.agents.items()):
|
|
385
|
+
if handle.process.returncode is None:
|
|
386
|
+
continue
|
|
387
|
+
self.agents.pop(agent_id, None)
|
|
388
|
+
self.agent_by_identity.pop(handle.identity, None)
|
|
389
|
+
await self.journals.mark_stopped(agent_id)
|
|
390
|
+
for job in self.jobs.values():
|
|
391
|
+
if (
|
|
392
|
+
job.agent_id == agent_id
|
|
393
|
+
and job.instance_id == handle.instance_id
|
|
394
|
+
and job.status
|
|
395
|
+
not in {
|
|
396
|
+
JobStatus.COMPLETED.value,
|
|
397
|
+
JobStatus.FAILED.value,
|
|
398
|
+
JobStatus.CANCELLED.value,
|
|
399
|
+
}
|
|
400
|
+
):
|
|
401
|
+
job.status = JobStatus.FAILED.value
|
|
402
|
+
job.error = f"Agent process exited with code {handle.process.returncode}."
|
|
403
|
+
sinapsis_logger.info("Agent %s process exited with code %s.", agent_id, handle.process.returncode)
|
|
404
|
+
|
|
405
|
+
async def _journal_gc_loop(self) -> None:
|
|
406
|
+
while True:
|
|
407
|
+
await asyncio.sleep(self.config.journal_gc_interval_seconds)
|
|
408
|
+
removed = await self.journals.purge_expired_stopped_agents()
|
|
409
|
+
if removed:
|
|
410
|
+
sinapsis_logger.info("Purged expired stopped-agent journals: %s", removed)
|
|
411
|
+
|
|
412
|
+
async def _terminate_agents(self) -> None:
|
|
413
|
+
for handle in tuple(self.agents.values()):
|
|
414
|
+
if handle.process.returncode is None:
|
|
415
|
+
handle.process.terminate()
|
|
416
|
+
await asyncio.gather(*(handle.process.wait() for handle in self.agents.values()), return_exceptions=True)
|
|
417
|
+
|
|
418
|
+
def _require_job(self, job_id: str) -> JobRecord:
|
|
419
|
+
job = self.jobs.get(job_id)
|
|
420
|
+
if job is None:
|
|
421
|
+
sinapsis_logger.error("Unknown job: %s.", job_id)
|
|
422
|
+
raise KeyError(f"Unknown job: {job_id}.")
|
|
423
|
+
return job
|
|
424
|
+
|
|
425
|
+
def _cache_command_result(self, command_id: str, result: dict[str, Any]) -> None:
|
|
426
|
+
self.command_results[command_id] = result
|
|
427
|
+
self.command_results.move_to_end(command_id)
|
|
428
|
+
while len(self.command_results) > SINAPSIS_API_COMMAND_CACHE_SIZE:
|
|
429
|
+
self.command_results.popitem(last=False)
|
|
430
|
+
|
|
431
|
+
def _track(self, task: asyncio.Task[Any]) -> None:
|
|
432
|
+
self._background_tasks.add(task)
|
|
433
|
+
task.add_done_callback(self._background_tasks.discard)
|
|
434
|
+
|
|
435
|
+
@staticmethod
|
|
436
|
+
def _error(error: RouterError, detail: str) -> dict[str, Any]:
|
|
437
|
+
return {"ok": False, "error": error.value, "detail": detail}
|
|
438
|
+
|
|
439
|
+
@staticmethod
|
|
440
|
+
def _job_dict(job: JobRecord) -> dict[str, Any]:
|
|
441
|
+
return {
|
|
442
|
+
"job_id": job.job_id,
|
|
443
|
+
"agent_id": job.agent_id,
|
|
444
|
+
"instance_id": job.instance_id,
|
|
445
|
+
"status": job.status,
|
|
446
|
+
"error": job.error,
|
|
447
|
+
}
|
|
448
|
+
|
|
449
|
+
@staticmethod
|
|
450
|
+
def _window_dict(window: Any) -> dict[str, Any]:
|
|
451
|
+
return {
|
|
452
|
+
"stream_id": window.stream_id,
|
|
453
|
+
"agent_id": window.agent_id,
|
|
454
|
+
"oldest_cursor": window.oldest_cursor,
|
|
455
|
+
"latest_cursor": window.latest_cursor,
|
|
456
|
+
"event_count": window.event_count,
|
|
457
|
+
"size_bytes": window.size_bytes,
|
|
458
|
+
}
|
|
459
|
+
|
|
460
|
+
def _replay_dict(self, replay: ReplayResult) -> dict[str, Any]:
|
|
461
|
+
return {
|
|
462
|
+
"ok": True,
|
|
463
|
+
"stream_id": replay.window.stream_id,
|
|
464
|
+
"agent_id": replay.window.agent_id,
|
|
465
|
+
"requested_after": replay.requested_after,
|
|
466
|
+
"effective_after": replay.effective_after,
|
|
467
|
+
"oldest_cursor": replay.window.oldest_cursor,
|
|
468
|
+
"latest_cursor": replay.window.latest_cursor,
|
|
469
|
+
"next_cursor": replay.next_cursor,
|
|
470
|
+
"truncated": replay.truncated,
|
|
471
|
+
"has_more": replay.has_more,
|
|
472
|
+
"event_count": len(replay.events),
|
|
473
|
+
}
|
|
474
|
+
|
|
475
|
+
|
|
476
|
+
async def main() -> None:
|
|
477
|
+
"""Run the Router process."""
|
|
478
|
+
await SinapsisAgentRouter().run()
|
|
479
|
+
|
|
480
|
+
|
|
481
|
+
if __name__ == "__main__":
|
|
482
|
+
asyncio.run(main())
|