sinapsis-api 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sinapsis_api/__init__.py +1 -0
- sinapsis_api/agent_router.py +482 -0
- sinapsis_api/agent_worker.py +240 -0
- sinapsis_api/api/__init__.py +1 -0
- sinapsis_api/api/healthcheck.py +25 -0
- sinapsis_api/api/main.py +40 -0
- sinapsis_api/api/routes/_common.py +85 -0
- sinapsis_api/api/routes/agents.py +91 -0
- sinapsis_api/api/routes/health.py +18 -0
- sinapsis_api/api/routes/jobs.py +81 -0
- sinapsis_api/client.py +135 -0
- sinapsis_api/constants.py +57 -0
- sinapsis_api/docker_runtime/__init__.py +42 -0
- sinapsis_api/docker_runtime/_file.py +71 -0
- sinapsis_api/docker_runtime/base.py +290 -0
- sinapsis_api/docker_runtime/image.py +83 -0
- sinapsis_api/docker_runtime/runtime.py +143 -0
- sinapsis_api/docker_runtime/services.py +127 -0
- sinapsis_api/env_vars/__init__.py +19 -0
- sinapsis_api/env_vars/env_vars.py +202 -0
- sinapsis_api/journal.py +298 -0
- sinapsis_api/models.py +30 -0
- sinapsis_api/onexth_runtime.py +68 -0
- sinapsis_api/protocol.py +106 -0
- sinapsis_api/telemetry.py +186 -0
- sinapsis_api-1.0.0.dist-info/METADATA +280 -0
- sinapsis_api-1.0.0.dist-info/RECORD +30 -0
- sinapsis_api-1.0.0.dist-info/WHEEL +4 -0
- sinapsis_api-1.0.0.dist-info/licenses/LICENSE-APACHE +202 -0
- sinapsis_api-1.0.0.dist-info/licenses/LICENSE-MIT +21 -0
sinapsis_api/journal.py
ADDED
|
@@ -0,0 +1,298 @@
|
|
|
1
|
+
"""Per-agent bounded event journals with stateless replay/live consumers."""
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
import time
|
|
5
|
+
from collections import deque
|
|
6
|
+
from dataclasses import dataclass, field
|
|
7
|
+
|
|
8
|
+
from sinapsis.logging import sinapsis_logger
|
|
9
|
+
|
|
10
|
+
from sinapsis_api.protocol import EventRecord
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@dataclass(frozen=True, slots=True)
|
|
14
|
+
class JournalWindow:
|
|
15
|
+
"""Current retained window for one agent stream."""
|
|
16
|
+
|
|
17
|
+
stream_id: str
|
|
18
|
+
agent_id: str
|
|
19
|
+
oldest_cursor: int
|
|
20
|
+
latest_cursor: int
|
|
21
|
+
event_count: int
|
|
22
|
+
size_bytes: int
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@dataclass(frozen=True, slots=True)
|
|
26
|
+
class ReplayResult:
|
|
27
|
+
"""A bounded replay result and its retained-window metadata."""
|
|
28
|
+
|
|
29
|
+
events: tuple[EventRecord, ...]
|
|
30
|
+
window: JournalWindow
|
|
31
|
+
requested_after: int
|
|
32
|
+
effective_after: int
|
|
33
|
+
truncated: bool
|
|
34
|
+
has_more: bool
|
|
35
|
+
next_cursor: int
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@dataclass(slots=True)
|
|
39
|
+
class Subscription:
|
|
40
|
+
"""Ephemeral live subscriber state; never persisted as consumer state."""
|
|
41
|
+
|
|
42
|
+
queue: asyncio.Queue[EventRecord]
|
|
43
|
+
after: int
|
|
44
|
+
job_id: str | None
|
|
45
|
+
closed: bool = False
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
@dataclass(slots=True)
|
|
49
|
+
class _AgentJournal:
|
|
50
|
+
stream_id: str
|
|
51
|
+
agent_id: str
|
|
52
|
+
max_events: int
|
|
53
|
+
max_bytes: int
|
|
54
|
+
retention_ns: int
|
|
55
|
+
events: deque[EventRecord] = field(default_factory=deque)
|
|
56
|
+
next_cursor: int = 1
|
|
57
|
+
size_bytes: int = 0
|
|
58
|
+
stopped_ns: int | None = None
|
|
59
|
+
subscriptions: dict[int, Subscription] = field(default_factory=dict)
|
|
60
|
+
next_subscription_id: int = 1
|
|
61
|
+
|
|
62
|
+
def prune(self, now_ns: int) -> None:
|
|
63
|
+
cutoff = now_ns - self.retention_ns
|
|
64
|
+
while self.events and (
|
|
65
|
+
len(self.events) > self.max_events or self.size_bytes > self.max_bytes or self.events[0].created_ns < cutoff
|
|
66
|
+
):
|
|
67
|
+
removed = self.events.popleft()
|
|
68
|
+
self.size_bytes -= removed.size_bytes
|
|
69
|
+
|
|
70
|
+
def window(self) -> JournalWindow:
|
|
71
|
+
oldest = self.events[0].cursor if self.events else self.next_cursor
|
|
72
|
+
return JournalWindow(
|
|
73
|
+
stream_id=self.stream_id,
|
|
74
|
+
agent_id=self.agent_id,
|
|
75
|
+
oldest_cursor=oldest,
|
|
76
|
+
latest_cursor=self.next_cursor - 1,
|
|
77
|
+
event_count=len(self.events),
|
|
78
|
+
size_bytes=self.size_bytes,
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
class AgentEventJournals:
|
|
83
|
+
"""Own independent rolling event streams keyed by logical agent ID."""
|
|
84
|
+
|
|
85
|
+
def __init__(
|
|
86
|
+
self,
|
|
87
|
+
*,
|
|
88
|
+
stream_id: str,
|
|
89
|
+
retention_seconds: float,
|
|
90
|
+
max_events: int,
|
|
91
|
+
max_bytes: int,
|
|
92
|
+
max_event_bytes: int,
|
|
93
|
+
stopped_agent_retention_seconds: float,
|
|
94
|
+
) -> None:
|
|
95
|
+
self.stream_id = stream_id
|
|
96
|
+
self._retention_ns = int(retention_seconds * 1_000_000_000)
|
|
97
|
+
self._max_events = max_events
|
|
98
|
+
self._max_bytes = max_bytes
|
|
99
|
+
self._max_event_bytes = max_event_bytes
|
|
100
|
+
self._stopped_retention_ns = int(stopped_agent_retention_seconds * 1_000_000_000)
|
|
101
|
+
self._journals: dict[str, _AgentJournal] = {}
|
|
102
|
+
self._lock = asyncio.Lock()
|
|
103
|
+
|
|
104
|
+
async def ensure_agent(self, agent_id: str) -> None:
|
|
105
|
+
"""Create an agent journal if it does not already exist."""
|
|
106
|
+
async with self._lock:
|
|
107
|
+
journal = self._journals.get(agent_id)
|
|
108
|
+
if journal is None:
|
|
109
|
+
self._journals[agent_id] = self._new_journal(agent_id)
|
|
110
|
+
else:
|
|
111
|
+
journal.stopped_ns = None
|
|
112
|
+
|
|
113
|
+
async def mark_stopped(self, agent_id: str) -> None:
|
|
114
|
+
"""Mark an agent stopped while preserving its rolling history."""
|
|
115
|
+
async with self._lock:
|
|
116
|
+
journal = self._journals.get(agent_id)
|
|
117
|
+
if journal is not None and journal.stopped_ns is None:
|
|
118
|
+
journal.stopped_ns = time.time_ns()
|
|
119
|
+
|
|
120
|
+
async def append(self, *, agent_id: str, instance_id: str, job_id: str | None, payload: bytes) -> EventRecord:
|
|
121
|
+
"""Append an opaque event and fan it out to this agent's subscribers."""
|
|
122
|
+
if len(payload) > self._max_event_bytes:
|
|
123
|
+
message = f"Event payload exceeds max size: {len(payload)} > {self._max_event_bytes}."
|
|
124
|
+
sinapsis_logger.error(message)
|
|
125
|
+
raise ValueError(message)
|
|
126
|
+
|
|
127
|
+
async with self._lock:
|
|
128
|
+
journal = self._journals.setdefault(agent_id, self._new_journal(agent_id))
|
|
129
|
+
now_ns = time.time_ns()
|
|
130
|
+
record = EventRecord(
|
|
131
|
+
stream_id=self.stream_id,
|
|
132
|
+
cursor=journal.next_cursor,
|
|
133
|
+
created_ns=now_ns,
|
|
134
|
+
agent_id=agent_id,
|
|
135
|
+
instance_id=instance_id,
|
|
136
|
+
job_id=job_id,
|
|
137
|
+
payload=payload,
|
|
138
|
+
)
|
|
139
|
+
journal.next_cursor += 1
|
|
140
|
+
journal.events.append(record)
|
|
141
|
+
journal.size_bytes += record.size_bytes
|
|
142
|
+
journal.prune(now_ns)
|
|
143
|
+
|
|
144
|
+
for subscription in tuple(journal.subscriptions.values()):
|
|
145
|
+
if subscription.closed or record.cursor <= subscription.after:
|
|
146
|
+
continue
|
|
147
|
+
if subscription.job_id is not None and record.job_id != subscription.job_id:
|
|
148
|
+
continue
|
|
149
|
+
try:
|
|
150
|
+
subscription.queue.put_nowait(record)
|
|
151
|
+
subscription.after = record.cursor
|
|
152
|
+
except asyncio.QueueFull:
|
|
153
|
+
subscription.closed = True
|
|
154
|
+
return record
|
|
155
|
+
|
|
156
|
+
async def replay(
|
|
157
|
+
self, *, agent_id: str, after: int, job_id: str | None, limit: int, max_bytes: int
|
|
158
|
+
) -> ReplayResult:
|
|
159
|
+
"""Return retained matching events, clamping an expired cursor to the window."""
|
|
160
|
+
async with self._lock:
|
|
161
|
+
journal = self._require(agent_id)
|
|
162
|
+
journal.prune(time.time_ns())
|
|
163
|
+
return self._replay_locked(journal, after, job_id, limit, max_bytes)
|
|
164
|
+
|
|
165
|
+
async def replay_through(
|
|
166
|
+
self,
|
|
167
|
+
*,
|
|
168
|
+
agent_id: str,
|
|
169
|
+
after: int,
|
|
170
|
+
through: int,
|
|
171
|
+
job_id: str | None,
|
|
172
|
+
limit: int,
|
|
173
|
+
max_bytes: int,
|
|
174
|
+
) -> ReplayResult:
|
|
175
|
+
"""Return one replay page capped at a previously captured live boundary."""
|
|
176
|
+
async with self._lock:
|
|
177
|
+
journal = self._require(agent_id)
|
|
178
|
+
journal.prune(time.time_ns())
|
|
179
|
+
return self._replay_locked(journal, after, job_id, limit, max_bytes, through=through)
|
|
180
|
+
|
|
181
|
+
async def subscribe(
|
|
182
|
+
self,
|
|
183
|
+
*,
|
|
184
|
+
agent_id: str,
|
|
185
|
+
after: int | None,
|
|
186
|
+
job_id: str | None,
|
|
187
|
+
queue_size: int,
|
|
188
|
+
replay_limit: int,
|
|
189
|
+
replay_max_bytes: int,
|
|
190
|
+
) -> tuple[int, ReplayResult | None, Subscription]:
|
|
191
|
+
"""Atomically establish a subscriber and optional replay boundary.
|
|
192
|
+
|
|
193
|
+
``after=None`` means live-only. Any integer, including zero, means replay
|
|
194
|
+
from that cursor within the retained rolling window and then follow live.
|
|
195
|
+
"""
|
|
196
|
+
async with self._lock:
|
|
197
|
+
journal = self._require(agent_id)
|
|
198
|
+
journal.prune(time.time_ns())
|
|
199
|
+
boundary = journal.next_cursor - 1
|
|
200
|
+
replay = None
|
|
201
|
+
if after is not None:
|
|
202
|
+
replay = self._replay_locked(journal, after, job_id, replay_limit, replay_max_bytes, through=boundary)
|
|
203
|
+
subscription = Subscription(queue=asyncio.Queue(maxsize=queue_size), after=boundary, job_id=job_id)
|
|
204
|
+
subscription_id = journal.next_subscription_id
|
|
205
|
+
journal.next_subscription_id += 1
|
|
206
|
+
journal.subscriptions[subscription_id] = subscription
|
|
207
|
+
return subscription_id, replay, subscription
|
|
208
|
+
|
|
209
|
+
async def unsubscribe(self, agent_id: str, subscription_id: int) -> None:
|
|
210
|
+
"""Remove an ephemeral live subscription."""
|
|
211
|
+
async with self._lock:
|
|
212
|
+
journal = self._journals.get(agent_id)
|
|
213
|
+
if journal is not None:
|
|
214
|
+
journal.subscriptions.pop(subscription_id, None)
|
|
215
|
+
|
|
216
|
+
async def bounds(self, agent_id: str) -> JournalWindow:
|
|
217
|
+
"""Return the current rolling-window bounds for an agent."""
|
|
218
|
+
async with self._lock:
|
|
219
|
+
journal = self._require(agent_id)
|
|
220
|
+
journal.prune(time.time_ns())
|
|
221
|
+
return journal.window()
|
|
222
|
+
|
|
223
|
+
async def purge_expired_stopped_agents(self) -> list[str]:
|
|
224
|
+
"""Remove stopped agent journals after their configured grace period."""
|
|
225
|
+
now_ns = time.time_ns()
|
|
226
|
+
removed: list[str] = []
|
|
227
|
+
async with self._lock:
|
|
228
|
+
for agent_id, journal in tuple(self._journals.items()):
|
|
229
|
+
journal.prune(now_ns)
|
|
230
|
+
if journal.stopped_ns is None or journal.subscriptions:
|
|
231
|
+
continue
|
|
232
|
+
if now_ns - journal.stopped_ns >= self._stopped_retention_ns:
|
|
233
|
+
self._journals.pop(agent_id, None)
|
|
234
|
+
removed.append(agent_id)
|
|
235
|
+
return removed
|
|
236
|
+
|
|
237
|
+
def _new_journal(self, agent_id: str) -> _AgentJournal:
|
|
238
|
+
return _AgentJournal(
|
|
239
|
+
stream_id=self.stream_id,
|
|
240
|
+
agent_id=agent_id,
|
|
241
|
+
max_events=self._max_events,
|
|
242
|
+
max_bytes=self._max_bytes,
|
|
243
|
+
retention_ns=self._retention_ns,
|
|
244
|
+
)
|
|
245
|
+
|
|
246
|
+
def _require(self, agent_id: str) -> _AgentJournal:
|
|
247
|
+
journal = self._journals.get(agent_id)
|
|
248
|
+
if journal is None:
|
|
249
|
+
message = f"Unknown agent event stream: {agent_id}."
|
|
250
|
+
sinapsis_logger.error(message)
|
|
251
|
+
raise KeyError(message)
|
|
252
|
+
return journal
|
|
253
|
+
|
|
254
|
+
@staticmethod
|
|
255
|
+
def _replay_locked(
|
|
256
|
+
journal: _AgentJournal,
|
|
257
|
+
after: int,
|
|
258
|
+
job_id: str | None,
|
|
259
|
+
limit: int,
|
|
260
|
+
max_bytes: int,
|
|
261
|
+
through: int | None = None,
|
|
262
|
+
) -> ReplayResult:
|
|
263
|
+
window = journal.window()
|
|
264
|
+
minimum_after = max(0, window.oldest_cursor - 1)
|
|
265
|
+
effective_after = max(after, minimum_after)
|
|
266
|
+
truncated = after < minimum_after
|
|
267
|
+
selected: list[EventRecord] = []
|
|
268
|
+
selected_bytes = 0
|
|
269
|
+
has_more = False
|
|
270
|
+
|
|
271
|
+
for event in journal.events:
|
|
272
|
+
if event.cursor <= effective_after:
|
|
273
|
+
continue
|
|
274
|
+
if through is not None and event.cursor > through:
|
|
275
|
+
break
|
|
276
|
+
if job_id is not None and event.job_id != job_id:
|
|
277
|
+
continue
|
|
278
|
+
if len(selected) >= limit or (selected and selected_bytes + event.size_bytes > max_bytes):
|
|
279
|
+
has_more = True
|
|
280
|
+
break
|
|
281
|
+
selected.append(event)
|
|
282
|
+
selected_bytes += event.size_bytes
|
|
283
|
+
|
|
284
|
+
next_cursor = selected[-1].cursor if selected else effective_after
|
|
285
|
+
if not has_more and through is None:
|
|
286
|
+
next_cursor = max(next_cursor, window.latest_cursor)
|
|
287
|
+
elif not has_more and through is not None:
|
|
288
|
+
next_cursor = max(next_cursor, through)
|
|
289
|
+
|
|
290
|
+
return ReplayResult(
|
|
291
|
+
events=tuple(selected),
|
|
292
|
+
window=window,
|
|
293
|
+
requested_after=after,
|
|
294
|
+
effective_after=effective_after,
|
|
295
|
+
truncated=truncated,
|
|
296
|
+
has_more=has_more,
|
|
297
|
+
next_cursor=next_cursor,
|
|
298
|
+
)
|
sinapsis_api/models.py
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
"""Router-owned runtime records."""
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
from dataclasses import dataclass, field
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
from sinapsis_api.constants import JobStatus
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
@dataclass(slots=True)
|
|
11
|
+
class AgentHandle:
|
|
12
|
+
"""A live OS process for one logical agent."""
|
|
13
|
+
|
|
14
|
+
agent_id: str
|
|
15
|
+
instance_id: str
|
|
16
|
+
process: asyncio.subprocess.Process
|
|
17
|
+
identity: bytes
|
|
18
|
+
agent_definition: dict[str, Any]
|
|
19
|
+
ready: asyncio.Event = field(default_factory=asyncio.Event)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@dataclass(slots=True)
|
|
23
|
+
class JobRecord:
|
|
24
|
+
"""Control-plane job state independent of opaque event semantics."""
|
|
25
|
+
|
|
26
|
+
job_id: str
|
|
27
|
+
agent_id: str
|
|
28
|
+
instance_id: str
|
|
29
|
+
status: str = JobStatus.QUEUED.value
|
|
30
|
+
error: str | None = None
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
"""Narrow integration boundary with the Sinapsis agent runtime."""
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
from typing import TYPE_CHECKING, Any
|
|
5
|
+
|
|
6
|
+
from sinapsis.agent import Agent, AgentMode, AgentSchema
|
|
7
|
+
from sinapsis.data_transport import DataContainer
|
|
8
|
+
from sinapsis.logging import sinapsis_logger
|
|
9
|
+
|
|
10
|
+
if TYPE_CHECKING:
|
|
11
|
+
from sinapsis_api.agent_worker import AgentContext
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def build_agent(agent_definition: dict[str, Any]) -> Agent:
|
|
15
|
+
"""Build a Sinapsis Agent from its validated schema definition."""
|
|
16
|
+
schema = AgentSchema.model_validate(agent_definition)
|
|
17
|
+
return Agent(schema)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
async def execute_agent(
|
|
21
|
+
agent: Agent,
|
|
22
|
+
payload: DataContainer | None,
|
|
23
|
+
context: "AgentContext",
|
|
24
|
+
*,
|
|
25
|
+
production: bool = False,
|
|
26
|
+
) -> None:
|
|
27
|
+
"""Execute the configured Sinapsis mode without blocking IPC handling."""
|
|
28
|
+
mode = agent.agent_attributes.agent_mode
|
|
29
|
+
if mode == AgentMode.LOOP_FOREVER:
|
|
30
|
+
if not production:
|
|
31
|
+
sinapsis_logger.error("LOOP_FOREVER agent invoked as a finite job.")
|
|
32
|
+
raise ValueError("LOOP_FOREVER agents must execute as production tasks.")
|
|
33
|
+
await asyncio.to_thread(agent, agent_mode=AgentMode.LOOP_FOREVER)
|
|
34
|
+
return
|
|
35
|
+
payload = payload if isinstance(payload, DataContainer) else DataContainer()
|
|
36
|
+
|
|
37
|
+
if production:
|
|
38
|
+
sinapsis_logger.error("Finite agent mode %s invoked as production.", mode)
|
|
39
|
+
raise ValueError(f"{mode} is a job execution mode, not a production mode.")
|
|
40
|
+
if not isinstance(payload, DataContainer):
|
|
41
|
+
sinapsis_logger.error("Job payload did not deserialize to DataContainer.")
|
|
42
|
+
raise TypeError("Job payload must deserialize to DataContainer.")
|
|
43
|
+
|
|
44
|
+
if mode == AgentMode.SINGLE:
|
|
45
|
+
result = await asyncio.to_thread(agent, payload, agent_mode=AgentMode.SINGLE)
|
|
46
|
+
if result is not None:
|
|
47
|
+
await context.send(result)
|
|
48
|
+
return
|
|
49
|
+
|
|
50
|
+
if mode == AgentMode.GENERATOR:
|
|
51
|
+
generator = agent(payload, agent_mode=AgentMode.GENERATOR)
|
|
52
|
+
sentinel = object()
|
|
53
|
+
|
|
54
|
+
def next_result() -> Any:
|
|
55
|
+
return next(generator, sentinel)
|
|
56
|
+
|
|
57
|
+
num_iterations = 0
|
|
58
|
+
while True:
|
|
59
|
+
result = await asyncio.to_thread(next_result)
|
|
60
|
+
print("calling next iteration ", num_iterations)
|
|
61
|
+
num_iterations += 1
|
|
62
|
+
|
|
63
|
+
if result is sentinel:
|
|
64
|
+
return
|
|
65
|
+
await context.send(result)
|
|
66
|
+
|
|
67
|
+
sinapsis_logger.error("Unsupported agent mode: %s.", mode)
|
|
68
|
+
raise ValueError(f"Unsupported agent mode: {mode}.")
|
sinapsis_api/protocol.py
ADDED
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
"""Transport framing for control messages and opaque event payloads."""
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
import msgpack
|
|
7
|
+
from sinapsis.logging import sinapsis_logger
|
|
8
|
+
|
|
9
|
+
PROTOCOL_VERSION = 1
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def pack_control(value: dict[str, Any]) -> bytes:
|
|
13
|
+
"""Serialize a transport control message."""
|
|
14
|
+
return msgpack.packb(value, use_bin_type=True)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def unpack_control(data: bytes | memoryview) -> dict[str, Any]:
|
|
18
|
+
"""Deserialize a transport control message."""
|
|
19
|
+
value = msgpack.unpackb(data, raw=False, strict_map_key=False)
|
|
20
|
+
if not isinstance(value, dict):
|
|
21
|
+
sinapsis_logger.error("Control message must be a map.")
|
|
22
|
+
raise ValueError("Control message must be a map.")
|
|
23
|
+
return value
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@dataclass(frozen=True, slots=True)
|
|
27
|
+
class EventRecord:
|
|
28
|
+
"""Router metadata plus an opaque serialized SSEEnvelope."""
|
|
29
|
+
|
|
30
|
+
stream_id: str
|
|
31
|
+
cursor: int
|
|
32
|
+
created_ns: int
|
|
33
|
+
agent_id: str
|
|
34
|
+
instance_id: str
|
|
35
|
+
job_id: str | None
|
|
36
|
+
payload: bytes
|
|
37
|
+
|
|
38
|
+
@property
|
|
39
|
+
def size_bytes(self) -> int:
|
|
40
|
+
"""Return the retained payload size in bytes."""
|
|
41
|
+
return len(self.payload)
|
|
42
|
+
|
|
43
|
+
def metadata(self) -> dict[str, Any]:
|
|
44
|
+
"""Return transport metadata without inspecting the payload."""
|
|
45
|
+
return {
|
|
46
|
+
"version": PROTOCOL_VERSION,
|
|
47
|
+
"stream_id": self.stream_id,
|
|
48
|
+
"cursor": self.cursor,
|
|
49
|
+
"created_ns": self.created_ns,
|
|
50
|
+
"agent_id": self.agent_id,
|
|
51
|
+
"instance_id": self.instance_id,
|
|
52
|
+
"job_id": self.job_id,
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
def zmq_frames(self) -> tuple[bytes, bytes]:
|
|
56
|
+
"""Return metadata and opaque payload frames."""
|
|
57
|
+
return pack_control(self.metadata()), self.payload
|
|
58
|
+
|
|
59
|
+
def websocket_frame(self) -> bytes:
|
|
60
|
+
"""Build a binary WebSocket frame from a MessagePack header and opaque payload.
|
|
61
|
+
|
|
62
|
+
The MessagePack metadata object is self-delimiting. The serialized
|
|
63
|
+
SSEEnvelope is appended byte-for-byte and is never decoded, encoded,
|
|
64
|
+
or included inside another serialization model.
|
|
65
|
+
"""
|
|
66
|
+
return pack_control(self.metadata()) + self.payload
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def split_websocket_frame(frame: bytes | memoryview) -> tuple[dict[str, Any], memoryview]:
|
|
70
|
+
"""Split one event frame without touching the serialized SSEEnvelope.
|
|
71
|
+
|
|
72
|
+
Exactly one MessagePack object is consumed from the start of ``frame``.
|
|
73
|
+
Every byte after that object is returned as the opaque event payload.
|
|
74
|
+
|
|
75
|
+
Args:
|
|
76
|
+
frame: Binary frame containing a MessagePack metadata header followed
|
|
77
|
+
immediately by a serialized SSEEnvelope.
|
|
78
|
+
|
|
79
|
+
Returns:
|
|
80
|
+
A tuple containing the decoded transport metadata and a zero-copy view
|
|
81
|
+
over the original serialized SSEEnvelope bytes.
|
|
82
|
+
|
|
83
|
+
Raises:
|
|
84
|
+
ValueError: If the frame does not start with a valid metadata map or
|
|
85
|
+
contains no event payload.
|
|
86
|
+
"""
|
|
87
|
+
view = memoryview(frame)
|
|
88
|
+
unpacker = msgpack.Unpacker(raw=False, strict_map_key=False)
|
|
89
|
+
unpacker.feed(view)
|
|
90
|
+
|
|
91
|
+
try:
|
|
92
|
+
metadata = next(unpacker)
|
|
93
|
+
except (StopIteration, msgpack.UnpackException, ValueError) as exception:
|
|
94
|
+
sinapsis_logger.exception("Invalid event frame metadata.")
|
|
95
|
+
raise ValueError("Invalid event frame metadata.") from exception
|
|
96
|
+
|
|
97
|
+
if not isinstance(metadata, dict):
|
|
98
|
+
sinapsis_logger.error("Event frame metadata must be a map.")
|
|
99
|
+
raise ValueError("Event frame metadata must be a map.")
|
|
100
|
+
|
|
101
|
+
metadata_end = unpacker.tell()
|
|
102
|
+
if metadata_end >= len(view):
|
|
103
|
+
sinapsis_logger.error("Event frame does not contain an SSEEnvelope payload.")
|
|
104
|
+
raise ValueError("Event frame does not contain an SSEEnvelope payload.")
|
|
105
|
+
|
|
106
|
+
return metadata, view[metadata_end:]
|
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
"""Opt-in, automatically delivered, container-local Sinapsis package inventory.
|
|
2
|
+
|
|
3
|
+
This module never inspects other distributions or execution inputs. Its SQLite
|
|
4
|
+
queue survives container restarts when /var/lib/sinapsis/telemetry is a volume.
|
|
5
|
+
Duplicate retries keep the original event UUID; only acknowledged events are
|
|
6
|
+
marked sent. No permanent installation or host identifier is created.
|
|
7
|
+
"""
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import json
|
|
11
|
+
import os
|
|
12
|
+
import re
|
|
13
|
+
import sqlite3
|
|
14
|
+
import sys
|
|
15
|
+
import time
|
|
16
|
+
import urllib.error
|
|
17
|
+
import urllib.parse
|
|
18
|
+
import urllib.request
|
|
19
|
+
from datetime import UTC, datetime
|
|
20
|
+
from importlib import metadata
|
|
21
|
+
from pathlib import Path
|
|
22
|
+
from uuid import uuid4
|
|
23
|
+
|
|
24
|
+
MAX_PACKAGES = 100
|
|
25
|
+
INTERVAL_SECONDS = 86400
|
|
26
|
+
DEFAULT_ENDPOINT = 'https://api.sinapsis.tech/api/v1/telemetry/events'
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def enabled() -> bool:
|
|
30
|
+
"""Require explicit container consent (independent of the host CLI)."""
|
|
31
|
+
return os.environ.get('SINAPSIS_TELEMETRY', '0').strip() == '1'
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def state_dir() -> Path:
|
|
35
|
+
return Path(os.environ.get('SINAPSIS_TELEMETRY_STATE_DIR', '/tmp/sinapsis/telemetry'))
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _connect() -> sqlite3.Connection:
|
|
39
|
+
folder = state_dir()
|
|
40
|
+
folder.mkdir(parents=True, exist_ok=True, mode=0o700)
|
|
41
|
+
path = folder / 'container_telemetry.sqlite3'
|
|
42
|
+
conn = sqlite3.connect(path, timeout=2)
|
|
43
|
+
if os.name == 'posix':
|
|
44
|
+
path.chmod(0o600)
|
|
45
|
+
conn.execute('CREATE TABLE IF NOT EXISTS events ('
|
|
46
|
+
'event_id TEXT PRIMARY KEY, timestamp TEXT NOT NULL, payload TEXT NOT NULL, '
|
|
47
|
+
'sent_at TEXT, lease_until REAL DEFAULT 0)')
|
|
48
|
+
conn.commit()
|
|
49
|
+
return conn
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def inventory() -> dict[str, object]:
|
|
53
|
+
"""Inspect only Sinapsis distribution metadata inside this container."""
|
|
54
|
+
distributions: dict[str, str] = {}
|
|
55
|
+
for dist in metadata.distributions():
|
|
56
|
+
name = re.sub(r'[-_.]+', '-', dist.metadata.get('Name', '')).casefold()
|
|
57
|
+
if name == 'sinapsis' or name.startswith('sinapsis-'):
|
|
58
|
+
distributions[name[:120]] = str(dist.version)[:64]
|
|
59
|
+
names = sorted(distributions)
|
|
60
|
+
return {
|
|
61
|
+
'schema_version': 1, 'event_id': str(uuid4()),
|
|
62
|
+
'timestamp': datetime.now(UTC).isoformat(timespec='seconds'),
|
|
63
|
+
'event_type': 'environment.inventory', 'source': 'local-api-container',
|
|
64
|
+
'environment': 'container', 'python': f'{sys.version_info.major}.{sys.version_info.minor}',
|
|
65
|
+
'distributions': [{'name': name, 'version': distributions[name]} for name in names[:MAX_PACKAGES]],
|
|
66
|
+
'distribution_count': len(names), 'truncated': len(names) > MAX_PACKAGES,
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def collect_inventory() -> bool:
|
|
71
|
+
"""Record at most one inventory per 24h, without blocking local execution."""
|
|
72
|
+
if not enabled():
|
|
73
|
+
return False
|
|
74
|
+
try:
|
|
75
|
+
conn = _connect()
|
|
76
|
+
try:
|
|
77
|
+
row = conn.execute('SELECT timestamp FROM events ORDER BY timestamp DESC LIMIT 1').fetchone()
|
|
78
|
+
if row and (datetime.now(UTC)-datetime.fromisoformat(row[0])).total_seconds() < INTERVAL_SECONDS:
|
|
79
|
+
return False
|
|
80
|
+
event = inventory()
|
|
81
|
+
with conn:
|
|
82
|
+
conn.execute('INSERT INTO events(event_id,timestamp,payload) VALUES(?,?,?)',
|
|
83
|
+
(event['event_id'], event['timestamp'], json.dumps(event, sort_keys=True)))
|
|
84
|
+
finally:
|
|
85
|
+
conn.close()
|
|
86
|
+
return True
|
|
87
|
+
except (OSError, ValueError, sqlite3.Error):
|
|
88
|
+
return False
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def publish_pending() -> int:
|
|
92
|
+
"""Automatically send one queue batch, acknowledging only persisted IDs."""
|
|
93
|
+
if not enabled():
|
|
94
|
+
return 0
|
|
95
|
+
endpoint = os.environ.get('SINAPSIS_TELEMETRY_ENDPOINT', DEFAULT_ENDPOINT)
|
|
96
|
+
parsed = urllib.parse.urlsplit(endpoint)
|
|
97
|
+
if parsed.scheme != 'https' or not parsed.netloc or parsed.username or parsed.password:
|
|
98
|
+
return 0
|
|
99
|
+
conn: sqlite3.Connection | None = None
|
|
100
|
+
ids: list[str] = []
|
|
101
|
+
acknowledged: set[str] = set()
|
|
102
|
+
try:
|
|
103
|
+
conn = _connect()
|
|
104
|
+
conn.execute('BEGIN IMMEDIATE')
|
|
105
|
+
rows = conn.execute('SELECT event_id,payload FROM events WHERE sent_at IS NULL '
|
|
106
|
+
'AND COALESCE(lease_until,0) <= ? ORDER BY timestamp LIMIT 40',
|
|
107
|
+
(time.time(),)).fetchall()
|
|
108
|
+
for event_id, _ in rows:
|
|
109
|
+
conn.execute('UPDATE events SET lease_until=? WHERE event_id=?', (time.time()+90, event_id))
|
|
110
|
+
conn.commit()
|
|
111
|
+
if not rows:
|
|
112
|
+
return 0
|
|
113
|
+
ids = [row[0] for row in rows]
|
|
114
|
+
events = [json.loads(row[1]) for row in rows]
|
|
115
|
+
while events:
|
|
116
|
+
body = json.dumps({'schema_version': 1, 'events': events}, separators=(',', ':')).encode()
|
|
117
|
+
if len(body) <= 65536:
|
|
118
|
+
break
|
|
119
|
+
events.pop()
|
|
120
|
+
if not events:
|
|
121
|
+
return 0
|
|
122
|
+
request = urllib.request.Request(endpoint, method='POST', data=body,
|
|
123
|
+
headers={'Content-Type': 'application/json'})
|
|
124
|
+
with urllib.request.urlopen(request, timeout=1.5) as response:
|
|
125
|
+
if response.status != 200:
|
|
126
|
+
return 0
|
|
127
|
+
result = json.loads(response.read(16384))
|
|
128
|
+
ack = result.get('acknowledged') if isinstance(result, dict) else None
|
|
129
|
+
if not isinstance(ack, list) or not all(isinstance(a, str) for a in ack):
|
|
130
|
+
return 0
|
|
131
|
+
acknowledged = set(ids).intersection(ack)
|
|
132
|
+
return len(acknowledged)
|
|
133
|
+
except (OSError, urllib.error.URLError, ValueError, TypeError, sqlite3.Error):
|
|
134
|
+
return 0
|
|
135
|
+
finally:
|
|
136
|
+
if conn is not None:
|
|
137
|
+
try:
|
|
138
|
+
with conn:
|
|
139
|
+
for event_id in ids:
|
|
140
|
+
if event_id in acknowledged:
|
|
141
|
+
conn.execute('UPDATE events SET sent_at=?,lease_until=0 WHERE event_id=?',
|
|
142
|
+
(datetime.now(UTC).isoformat(), event_id))
|
|
143
|
+
else:
|
|
144
|
+
conn.execute('UPDATE events SET lease_until=0 WHERE event_id=?', (event_id,))
|
|
145
|
+
except sqlite3.Error:
|
|
146
|
+
pass
|
|
147
|
+
conn.close()
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def info() -> dict[str, object]:
|
|
151
|
+
"""Return exact last sent payload and queue counts for local inspection."""
|
|
152
|
+
if not state_dir().joinpath('container_telemetry.sqlite3').exists():
|
|
153
|
+
return {'enabled': enabled(), 'pending': 0, 'sent': 0, 'last_sent': None}
|
|
154
|
+
try:
|
|
155
|
+
conn = _connect()
|
|
156
|
+
pending = conn.execute('SELECT count(*) FROM events WHERE sent_at IS NULL').fetchone()[0]
|
|
157
|
+
sent = conn.execute('SELECT count(*) FROM events WHERE sent_at IS NOT NULL').fetchone()[0]
|
|
158
|
+
last = conn.execute('SELECT sent_at,payload FROM events WHERE sent_at IS NOT NULL '
|
|
159
|
+
'ORDER BY sent_at DESC LIMIT 1').fetchone()
|
|
160
|
+
conn.close()
|
|
161
|
+
return {'enabled': enabled(), 'pending': pending, 'sent': sent,
|
|
162
|
+
'last_sent': {'acknowledged_at': last[0], 'payload': json.loads(last[1])} if last else None}
|
|
163
|
+
except (OSError, ValueError, sqlite3.Error):
|
|
164
|
+
return {'enabled': enabled(), 'error': 'Telemetry journal unavailable'}
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def main() -> None:
|
|
168
|
+
"""Provide transparent local control: info, preview, send or clear."""
|
|
169
|
+
action = sys.argv[1] if len(sys.argv) > 1 else 'info'
|
|
170
|
+
if action == 'info':
|
|
171
|
+
print(json.dumps(info(), indent=2))
|
|
172
|
+
print('Collected: Sinapsis package names/versions, Python version, time, event UUID.')
|
|
173
|
+
print('Never collected: environment variables, paths, execution data or arguments.')
|
|
174
|
+
elif action == 'preview':
|
|
175
|
+
print(json.dumps(inventory(), indent=2))
|
|
176
|
+
elif action == 'send':
|
|
177
|
+
print(f'Acknowledged {publish_pending()} events')
|
|
178
|
+
elif action == 'clear':
|
|
179
|
+
state_dir().joinpath('container_telemetry.sqlite3').unlink(missing_ok=True)
|
|
180
|
+
print('Cleared local container telemetry')
|
|
181
|
+
else:
|
|
182
|
+
raise SystemExit('Usage: python -m sinapsis_api.telemetry [info|preview|send|clear]')
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
if __name__ == '__main__':
|
|
186
|
+
main()
|