monkeybot 2.1.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (178) hide show
  1. monkeybot/__init__.py +3 -0
  2. monkeybot/cli/__init__.py +3 -0
  3. monkeybot/cli/__main__.py +8 -0
  4. monkeybot/cli/audio_io.py +8 -0
  5. monkeybot/cli/gateway_manager.py +17 -0
  6. monkeybot/cli/main.py +22 -0
  7. monkeybot/cli/push_to_talk.py +12 -0
  8. monkeybot/cli/realtime_client.py +13 -0
  9. monkeybot/core/__init__.py +19 -0
  10. monkeybot/core/attachments/__init__.py +22 -0
  11. monkeybot/core/attachments/catalog.py +62 -0
  12. monkeybot/core/attachments/config.py +52 -0
  13. monkeybot/core/attachments/freeze.py +158 -0
  14. monkeybot/core/attachments/resolve.py +70 -0
  15. monkeybot/core/attachments/store.py +180 -0
  16. monkeybot/core/attachments/text.py +72 -0
  17. monkeybot/core/attachments/tools.py +54 -0
  18. monkeybot/core/bootstrap.py +242 -0
  19. monkeybot/core/config/__init__.py +71 -0
  20. monkeybot/core/config/realtime_config.py +150 -0
  21. monkeybot/core/config/runtime_env.py +262 -0
  22. monkeybot/core/config/settings.py +341 -0
  23. monkeybot/core/config/validation.py +249 -0
  24. monkeybot/core/config/yaml_loader.py +45 -0
  25. monkeybot/core/context/__init__.py +781 -0
  26. monkeybot/core/context/campaign_context.py +8 -0
  27. monkeybot/core/context/common.py +14 -0
  28. monkeybot/core/context/curator.py +255 -0
  29. monkeybot/core/context/epoch.py +226 -0
  30. monkeybot/core/context/memory_prompt.py +222 -0
  31. monkeybot/core/context/tool_output_policy.py +270 -0
  32. monkeybot/core/context/tool_result_ingress.py +290 -0
  33. monkeybot/core/context/tool_shapers.py +361 -0
  34. monkeybot/core/hooks/__init__.py +261 -0
  35. monkeybot/core/llm/__init__.py +4 -0
  36. monkeybot/core/llm/provider.py +296 -0
  37. monkeybot/core/llm/realtime_provider.py +203 -0
  38. monkeybot/core/llm/usage.py +57 -0
  39. monkeybot/core/logging_utils.py +24 -0
  40. monkeybot/core/mcp/__init__.py +1 -0
  41. monkeybot/core/mcp/mcp_client.py +1215 -0
  42. monkeybot/core/mcp/ports_mcp.py +109 -0
  43. monkeybot/core/memory/__init__.py +24 -0
  44. monkeybot/core/memory/hook.py +413 -0
  45. monkeybot/core/memory/index_format.py +104 -0
  46. monkeybot/core/memory/integrity.py +180 -0
  47. monkeybot/core/memory/organizer.py +270 -0
  48. monkeybot/core/memory/storage_ops.py +139 -0
  49. monkeybot/core/memory/subsystem.py +91 -0
  50. monkeybot/core/messages/__init__.py +16 -0
  51. monkeybot/core/messages/convert_provider.py +41 -0
  52. monkeybot/core/messages/tool_integrity.py +262 -0
  53. monkeybot/core/messages/transform_context.py +84 -0
  54. monkeybot/core/path_safety.py +11 -0
  55. monkeybot/core/persistence/__init__.py +17 -0
  56. monkeybot/core/persistence/backends.py +236 -0
  57. monkeybot/core/persistence/db.py +28 -0
  58. monkeybot/core/persistence/durable_runs.py +286 -0
  59. monkeybot/core/persistence/firestore.py +658 -0
  60. monkeybot/core/persistence/firestore_scheduled_loops.py +336 -0
  61. monkeybot/core/persistence/history.py +156 -0
  62. monkeybot/core/persistence/postgres.py +895 -0
  63. monkeybot/core/persistence/runs.py +76 -0
  64. monkeybot/core/persistence/scheduled_loops.py +435 -0
  65. monkeybot/core/persistence/session_turn_locks.py +94 -0
  66. monkeybot/core/persistence/sqlite.py +218 -0
  67. monkeybot/core/persistence/sqlite_backend.py +74 -0
  68. monkeybot/core/persistence/thread_summary.py +61 -0
  69. monkeybot/core/persistence/transcript.py +194 -0
  70. monkeybot/core/persistence/usage.py +149 -0
  71. monkeybot/core/prompts/__init__.py +1 -0
  72. monkeybot/core/prompts/harness_prompt.py +197 -0
  73. monkeybot/core/prompts/prompt.py +215 -0
  74. monkeybot/core/runtime/__init__.py +1 -0
  75. monkeybot/core/runtime/context_budget.py +267 -0
  76. monkeybot/core/runtime/events.py +819 -0
  77. monkeybot/core/runtime/input_admission.py +154 -0
  78. monkeybot/core/runtime/loop.py +2374 -0
  79. monkeybot/core/runtime/provider_stream_mapper.py +159 -0
  80. monkeybot/core/runtime/realtime_loop.py +654 -0
  81. monkeybot/core/runtime/utterance_buffer.py +179 -0
  82. monkeybot/core/subagents/__init__.py +1 -0
  83. monkeybot/core/subagents/subagent_proto.py +331 -0
  84. monkeybot/core/subagents/subagent_worker.py +441 -0
  85. monkeybot/core/subagents/worker_pool.py +403 -0
  86. monkeybot/core/testing/__init__.py +1 -0
  87. monkeybot/core/testing/mocks_provider.py +86 -0
  88. monkeybot/core/testing/mocks_realtime_provider.py +137 -0
  89. monkeybot/core/tools/__init__.py +1 -0
  90. monkeybot/core/tools/core_tool_executor.py +1548 -0
  91. monkeybot/core/tools/inspector.py +226 -0
  92. monkeybot/core/tools/loop_inspector.py +45 -0
  93. monkeybot/core/tools/patch.py +480 -0
  94. monkeybot/core/tools/permission.py +284 -0
  95. monkeybot/core/tools/sandbox_executor.py +255 -0
  96. monkeybot/core/tools/spill_inventory.py +35 -0
  97. monkeybot/core/tools/terminal.py +381 -0
  98. monkeybot/core/tools/text_normalize.py +25 -0
  99. monkeybot/core/tools/types.py +33 -0
  100. monkeybot/core/tools/workspace_service.py +710 -0
  101. monkeybot/core/tools/workspace_tools.py +116 -0
  102. monkeybot/core/types/__init__.py +1 -0
  103. monkeybot/core/types/content_blocks.py +644 -0
  104. monkeybot/core/types/interfaces.py +156 -0
  105. monkeybot/core/types/types_tools.py +29 -0
  106. monkeybot/core/workspace/__init__.py +8 -0
  107. monkeybot/core/workspace/factory.py +45 -0
  108. monkeybot/core/workspace/gcs.py +130 -0
  109. monkeybot/core/workspace/local.py +162 -0
  110. monkeybot/core/workspace/protocol.py +45 -0
  111. monkeybot/core/workspace/s3.py +151 -0
  112. monkeybot/core/workspace_layout.py +27 -0
  113. monkeybot/gateway/__init__.py +1 -0
  114. monkeybot/gateway/bootstrap.py +18 -0
  115. monkeybot/gateway/main.py +47 -0
  116. monkeybot/gateway/realtime/__init__.py +31 -0
  117. monkeybot/gateway/realtime/app.py +321 -0
  118. monkeybot/gateway/realtime/deps.py +52 -0
  119. monkeybot/gateway/realtime/errors.py +81 -0
  120. monkeybot/gateway/realtime/guardrails.py +88 -0
  121. monkeybot/gateway/realtime/manager.py +77 -0
  122. monkeybot/gateway/realtime/metrics.py +144 -0
  123. monkeybot/gateway/realtime/routes.py +864 -0
  124. monkeybot/gateway/realtime/session.py +232 -0
  125. monkeybot/gateway/realtime/wire.py +412 -0
  126. monkeybot/gateway/realtime_main.py +49 -0
  127. monkeybot/gateway/sse/__init__.py +1 -0
  128. monkeybot/gateway/sse/app.py +733 -0
  129. monkeybot/gateway/sse/loop_port.py +31 -0
  130. monkeybot/gateway/sse/models.py +177 -0
  131. monkeybot/gateway/sse/reply_body.py +91 -0
  132. monkeybot/gateway/sse/routes.py +1101 -0
  133. monkeybot/gateway/sse/scheduler_routes.py +200 -0
  134. monkeybot/gateway/sse/scheduler_wiring.py +96 -0
  135. monkeybot/gateway/sse/session_bus.py +226 -0
  136. monkeybot/gateway/sse/sse.py +46 -0
  137. monkeybot/gateway/sse/workspace_layout.py +7 -0
  138. monkeybot/observability/__init__.py +220 -0
  139. monkeybot/observability/_state.py +10 -0
  140. monkeybot/observability/instrumentation.py +153 -0
  141. monkeybot/observability/propagation.py +65 -0
  142. monkeybot/observability/spans.py +455 -0
  143. monkeybot/providers/__init__.py +19 -0
  144. monkeybot/providers/_openai_compat.py +450 -0
  145. monkeybot/providers/_utils.py +473 -0
  146. monkeybot/providers/bedrock.py +145 -0
  147. monkeybot/providers/claude.py +125 -0
  148. monkeybot/providers/gemini.py +677 -0
  149. monkeybot/providers/gemini_live.py +398 -0
  150. monkeybot/providers/huggingface.py +129 -0
  151. monkeybot/providers/nvidia.py +104 -0
  152. monkeybot/providers/ollama.py +152 -0
  153. monkeybot/providers/openai.py +127 -0
  154. monkeybot/providers/pricing.py +60 -0
  155. monkeybot/providers/sampling.py +44 -0
  156. monkeybot/providers/vertex_claude.py +148 -0
  157. monkeybot/scaffold/__init__.py +33 -0
  158. monkeybot/scheduler/__init__.py +13 -0
  159. monkeybot/scheduler/__main__.py +4 -0
  160. monkeybot/scheduler/engine.py +333 -0
  161. monkeybot/scheduler/http_invoker.py +61 -0
  162. monkeybot/scheduler/interval.py +77 -0
  163. monkeybot/scheduler/tick_result.py +34 -0
  164. monkeybot/scheduler/worker.py +87 -0
  165. monkeybot/subagents/__init__.py +1 -0
  166. monkeybot/subagents/worker/__init__.py +1 -0
  167. monkeybot/subagents/worker/__main__.py +22 -0
  168. monkeybot/web_search/__init__.py +82 -0
  169. monkeybot/web_search/backends/__init__.py +5 -0
  170. monkeybot/web_search/backends/duckduckgo.py +32 -0
  171. monkeybot/web_search/backends/firecrawl.py +43 -0
  172. monkeybot/web_search/backends/tavily.py +45 -0
  173. monkeybot/web_search/protocol.py +25 -0
  174. monkeybot/web_search/tool.py +56 -0
  175. monkeybot-2.1.1.dist-info/METADATA +318 -0
  176. monkeybot-2.1.1.dist-info/RECORD +178 -0
  177. monkeybot-2.1.1.dist-info/WHEEL +4 -0
  178. monkeybot-2.1.1.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,895 @@
1
+ """PostgreSQL storage backend (requires ``pip install 'monkeybot[postgres]'``)."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ import logging
7
+ import os
8
+ import time
9
+ from typing import cast
10
+
11
+ import asyncpg
12
+
13
+ from monkeybot.core.llm.provider import Message, Role
14
+ from monkeybot.core.llm.usage import Usage, UsageSummary
15
+ from monkeybot.core.persistence.durable_runs import (
16
+ _SUBAGENT_COLUMNS,
17
+ SubagentEnvelope,
18
+ SubagentRunRow,
19
+ _tuple_to_run_row,
20
+ )
21
+ from monkeybot.core.persistence.scheduled_loops import (
22
+ ScheduledLoopCreate,
23
+ ScheduledLoopRow,
24
+ _SCHEDULED_LOOP_COLUMNS,
25
+ _loop_id_from_create,
26
+ _row_from_tuple,
27
+ validate_loop_guards,
28
+ )
29
+ from monkeybot.core.persistence.thread_summary import ChatThreadSummary, preview_from_content_blob
30
+ from monkeybot.core.types.content_blocks import ContentBlock
31
+
32
+ logger = logging.getLogger(__name__)
33
+
34
+ _VALID_ROLES: tuple[str, ...] = ("user", "assistant", "system")
35
+
36
+ _SCHEMA_DDLS: tuple[str, ...] = (
37
+ """CREATE TABLE IF NOT EXISTS conversation_history (
38
+ id BIGSERIAL PRIMARY KEY,
39
+ thread_id TEXT NOT NULL,
40
+ role TEXT NOT NULL,
41
+ content TEXT NOT NULL,
42
+ created_at BIGINT NOT NULL
43
+ )""",
44
+ """CREATE TABLE IF NOT EXISTS subagent_runs (
45
+ run_id TEXT PRIMARY KEY,
46
+ parent_run_id TEXT,
47
+ script TEXT NOT NULL,
48
+ envelope_json TEXT NOT NULL,
49
+ status TEXT NOT NULL,
50
+ result_json TEXT,
51
+ error_json TEXT,
52
+ started_at BIGINT,
53
+ finished_at BIGINT,
54
+ scratch_dir TEXT,
55
+ worker_id TEXT,
56
+ claimed_at BIGINT
57
+ )""",
58
+ """CREATE TABLE IF NOT EXISTS turn_usage (
59
+ id BIGSERIAL PRIMARY KEY,
60
+ thread_id TEXT NOT NULL,
61
+ run_id TEXT,
62
+ model TEXT NOT NULL,
63
+ input_tokens INTEGER NOT NULL,
64
+ output_tokens INTEGER NOT NULL,
65
+ cached_tokens INTEGER NOT NULL,
66
+ cost_usd DOUBLE PRECISION NOT NULL,
67
+ duration_ms INTEGER NOT NULL,
68
+ created_at BIGINT NOT NULL,
69
+ context_json TEXT,
70
+ estimated_prompt_tokens INTEGER NOT NULL DEFAULT 0,
71
+ cache_read_tokens INTEGER NOT NULL DEFAULT 0,
72
+ cache_creation_tokens INTEGER NOT NULL DEFAULT 0
73
+ )""",
74
+ "CREATE INDEX IF NOT EXISTS idx_history_thread ON conversation_history(thread_id, created_at)",
75
+ "CREATE INDEX IF NOT EXISTS idx_runs_parent ON subagent_runs(parent_run_id)",
76
+ "CREATE INDEX IF NOT EXISTS idx_runs_status ON subagent_runs(status) WHERE status IN ('pending','running')",
77
+ "CREATE INDEX IF NOT EXISTS idx_usage_thread ON turn_usage(thread_id, created_at)",
78
+ "CREATE INDEX IF NOT EXISTS idx_usage_cost ON turn_usage(created_at)",
79
+ "ALTER TABLE turn_usage ADD COLUMN IF NOT EXISTS cache_read_tokens INTEGER NOT NULL DEFAULT 0",
80
+ "ALTER TABLE turn_usage ADD COLUMN IF NOT EXISTS cache_creation_tokens INTEGER NOT NULL DEFAULT 0",
81
+ "ALTER TABLE subagent_runs ADD COLUMN IF NOT EXISTS worker_id TEXT",
82
+ "ALTER TABLE subagent_runs ADD COLUMN IF NOT EXISTS claimed_at BIGINT",
83
+ """CREATE TABLE IF NOT EXISTS scheduled_loops (
84
+ loop_id TEXT PRIMARY KEY,
85
+ session_id TEXT NOT NULL,
86
+ status TEXT NOT NULL,
87
+ prompt TEXT NOT NULL,
88
+ interval_ms BIGINT NOT NULL,
89
+ max_ticks INTEGER,
90
+ max_runtime_ms BIGINT,
91
+ skip_if_busy INTEGER NOT NULL DEFAULT 1,
92
+ tick_index INTEGER NOT NULL DEFAULT 0,
93
+ next_tick_at_ms BIGINT NOT NULL,
94
+ started_at_ms BIGINT NOT NULL,
95
+ last_tick_at_ms BIGINT,
96
+ last_error TEXT,
97
+ stop_reason TEXT,
98
+ tick_in_flight INTEGER NOT NULL DEFAULT 0,
99
+ worker_id TEXT,
100
+ claimed_at_ms BIGINT
101
+ )""",
102
+ """CREATE INDEX IF NOT EXISTS idx_scheduled_loops_due
103
+ ON scheduled_loops(status, tick_in_flight, next_tick_at_ms)
104
+ WHERE status = 'active'""",
105
+ """CREATE TABLE IF NOT EXISTS session_turn_locks (
106
+ session_id TEXT PRIMARY KEY,
107
+ request_id TEXT,
108
+ claimed_at_ms BIGINT
109
+ )""",
110
+ )
111
+
112
+
113
+ async def _apply_schema(pool: asyncpg.Pool) -> None:
114
+ async with pool.acquire() as conn:
115
+ for ddl in _SCHEMA_DDLS:
116
+ await conn.execute(ddl)
117
+
118
+
119
+ class PostgresHistoryStore:
120
+ """asyncpg-backed conversation history store."""
121
+
122
+ def __init__(self, pool: asyncpg.Pool) -> None:
123
+ self._pool = pool
124
+
125
+ async def _insert_message(self, conn: asyncpg.Connection, thread_id: str, message: Message) -> None:
126
+ role = message.role
127
+ if role not in _VALID_ROLES:
128
+ raise ValueError(f"invalid role: {role!r}")
129
+ payload = json.dumps(
130
+ [b.to_dict() for b in message.content],
131
+ separators=(",", ":"),
132
+ ensure_ascii=False,
133
+ )
134
+ created_at = int(time.time() * 1000)
135
+ await conn.execute(
136
+ """
137
+ INSERT INTO conversation_history(thread_id, role, content, created_at)
138
+ VALUES ($1, $2, $3, $4)
139
+ """,
140
+ thread_id,
141
+ message.role,
142
+ payload,
143
+ created_at,
144
+ )
145
+
146
+ async def append(self, thread_id: str, message: Message) -> None:
147
+ async with self._pool.acquire() as conn:
148
+ await self._insert_message(conn, thread_id, message)
149
+
150
+ async def load(self, thread_id: str, limit: int = 100) -> list[Message]:
151
+ async with self._pool.acquire() as conn:
152
+ rows = await conn.fetch(
153
+ """
154
+ SELECT id, role, content
155
+ FROM conversation_history
156
+ WHERE thread_id = $1
157
+ ORDER BY created_at DESC, id DESC
158
+ LIMIT $2
159
+ """,
160
+ thread_id,
161
+ limit,
162
+ )
163
+ rows_chrono = list(reversed(rows))
164
+ out: list[Message] = []
165
+ for row in rows_chrono:
166
+ row_id = int(row["id"])
167
+ role = row["role"]
168
+ content_blob = row["content"]
169
+ try:
170
+ raw = json.loads(content_blob)
171
+ if not isinstance(raw, list):
172
+ raise ValueError("stored content must be a JSON array")
173
+ blocks = [ContentBlock.from_dict(b) for b in raw]
174
+ except (json.JSONDecodeError, ValueError, TypeError) as exc:
175
+ logger.error(
176
+ "Unparseable history row id=%s thread_id=%s",
177
+ row_id,
178
+ thread_id,
179
+ exc_info=True,
180
+ )
181
+ raise ValueError(f"history row {row_id} unparseable: {exc}") from exc
182
+ if role not in _VALID_ROLES:
183
+ raise ValueError(f"history row {row_id} has invalid role: {role!r}")
184
+ out.append(Message(role=cast(Role, role), content=blocks))
185
+ return out
186
+
187
+ async def clear(self, thread_id: str) -> None:
188
+ async with self._pool.acquire() as conn:
189
+ await conn.execute(
190
+ "DELETE FROM conversation_history WHERE thread_id = $1", thread_id
191
+ )
192
+
193
+ async def reset(self, thread_id: str, messages: list[Message]) -> None:
194
+ """Replace thread history atomically (single transaction)."""
195
+ async with self._pool.acquire() as conn, conn.transaction():
196
+ await conn.execute(
197
+ "DELETE FROM conversation_history WHERE thread_id = $1", thread_id
198
+ )
199
+ for msg in messages:
200
+ await self._insert_message(conn, thread_id, msg)
201
+
202
+ async def list_threads(self, limit: int = 50) -> list[ChatThreadSummary]:
203
+ cap = max(1, min(limit, 200))
204
+ async with self._pool.acquire() as conn:
205
+ rows = await conn.fetch(
206
+ """
207
+ SELECT
208
+ h.thread_id,
209
+ MAX(h.created_at) AS last_message_at,
210
+ COUNT(*)::int AS message_count,
211
+ (
212
+ SELECT h2.content
213
+ FROM conversation_history h2
214
+ WHERE h2.thread_id = h.thread_id
215
+ ORDER BY h2.created_at DESC, h2.id DESC
216
+ LIMIT 1
217
+ ) AS last_content
218
+ FROM conversation_history h
219
+ GROUP BY h.thread_id
220
+ ORDER BY last_message_at DESC
221
+ LIMIT $1
222
+ """,
223
+ cap,
224
+ )
225
+ out: list[ChatThreadSummary] = []
226
+ for row in rows:
227
+ preview = preview_from_content_blob(str(row["last_content"] or ""))
228
+ out.append(
229
+ ChatThreadSummary(
230
+ thread_id=str(row["thread_id"]),
231
+ last_message_at=int(row["last_message_at"]),
232
+ message_count=int(row["message_count"]),
233
+ preview=preview or "(empty)",
234
+ )
235
+ )
236
+ return out
237
+
238
+
239
+ class PostgresUsageStore:
240
+ """asyncpg-backed usage store."""
241
+
242
+ def __init__(self, pool: asyncpg.Pool) -> None:
243
+ self._pool = pool
244
+
245
+ async def record(
246
+ self,
247
+ thread_id: str,
248
+ model: str,
249
+ usage: Usage,
250
+ run_id: str | None = None,
251
+ *,
252
+ context_json: str | None = None,
253
+ ) -> None:
254
+ now_ms = int(time.time() * 1000)
255
+ async with self._pool.acquire() as conn:
256
+ await conn.execute(
257
+ """
258
+ INSERT INTO turn_usage(
259
+ thread_id, run_id, model,
260
+ input_tokens, output_tokens, cached_tokens,
261
+ cost_usd, duration_ms, created_at, context_json,
262
+ estimated_prompt_tokens, cache_read_tokens, cache_creation_tokens
263
+ )
264
+ VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)
265
+ """,
266
+ thread_id,
267
+ run_id,
268
+ model,
269
+ usage.input_tokens,
270
+ usage.output_tokens,
271
+ usage.cached_tokens,
272
+ usage.cost_usd,
273
+ usage.duration_ms,
274
+ now_ms,
275
+ context_json,
276
+ usage.estimated_prompt_tokens,
277
+ usage.cache_read_tokens,
278
+ usage.cache_creation_tokens,
279
+ )
280
+
281
+ async def summary(
282
+ self,
283
+ thread_id: str | None = None,
284
+ since_ms: int | None = None,
285
+ ) -> UsageSummary:
286
+ clauses: list[str] = []
287
+ params: list[object] = []
288
+ idx = 1
289
+ if thread_id is not None:
290
+ clauses.append(f"thread_id = ${idx}")
291
+ params.append(thread_id)
292
+ idx += 1
293
+ if since_ms is not None:
294
+ clauses.append(f"created_at >= ${idx}")
295
+ params.append(since_ms)
296
+ idx += 1
297
+ where_sql = ("WHERE " + " AND ".join(clauses)) if clauses else ""
298
+
299
+ sql = f"""
300
+ SELECT
301
+ COUNT(*) AS turns,
302
+ COALESCE(SUM(input_tokens), 0) AS input_tokens,
303
+ COALESCE(SUM(output_tokens), 0) AS output_tokens,
304
+ COALESCE(SUM(cached_tokens), 0) AS cached_tokens,
305
+ COALESCE(SUM(cost_usd), 0.0) AS cost_usd,
306
+ MIN(created_at) AS period_start_ms,
307
+ MAX(created_at) AS period_end_ms,
308
+ COALESCE(SUM(cache_read_tokens), 0) AS cache_read_tokens,
309
+ COALESCE(SUM(cache_creation_tokens), 0) AS cache_creation_tokens
310
+ FROM turn_usage
311
+ {where_sql}
312
+ """
313
+
314
+ async with self._pool.acquire() as conn:
315
+ row = await conn.fetchrow(sql, *params)
316
+
317
+ if row is None:
318
+ raise RuntimeError("usage summary query returned no row")
319
+
320
+ turns = int(row["turns"])
321
+ if turns == 0:
322
+ return UsageSummary(
323
+ turns=0,
324
+ input_tokens=0,
325
+ output_tokens=0,
326
+ cached_tokens=0,
327
+ cost_usd=0.0,
328
+ period_start_ms=None,
329
+ period_end_ms=None,
330
+ last_prompt_tokens=0,
331
+ last_estimated_prompt_tokens=0,
332
+ cache_read_tokens=0,
333
+ cache_creation_tokens=0,
334
+ )
335
+
336
+ last_pt = 0
337
+ last_est = 0
338
+ if thread_id is not None:
339
+ lp_clauses: list[str] = ["thread_id = $1"]
340
+ lp_params: list[object] = [thread_id]
341
+ if since_ms is not None:
342
+ lp_clauses.append("created_at >= $2")
343
+ lp_params.append(since_ms)
344
+ lp_where = "WHERE " + " AND ".join(lp_clauses)
345
+ async with self._pool.acquire() as conn:
346
+ row2 = await conn.fetchrow(
347
+ f"""
348
+ SELECT input_tokens, estimated_prompt_tokens FROM turn_usage
349
+ {lp_where}
350
+ ORDER BY created_at DESC
351
+ LIMIT 1
352
+ """,
353
+ *lp_params,
354
+ )
355
+ if row2 is not None:
356
+ last_pt = int(row2["input_tokens"])
357
+ last_est = int(row2["estimated_prompt_tokens"])
358
+
359
+ return UsageSummary(
360
+ turns=turns,
361
+ input_tokens=int(row["input_tokens"]),
362
+ output_tokens=int(row["output_tokens"]),
363
+ cached_tokens=int(row["cached_tokens"]),
364
+ cost_usd=float(row["cost_usd"]),
365
+ period_start_ms=int(row["period_start_ms"]) if row["period_start_ms"] is not None else None,
366
+ period_end_ms=int(row["period_end_ms"]) if row["period_end_ms"] is not None else None,
367
+ last_prompt_tokens=last_pt,
368
+ last_estimated_prompt_tokens=last_est,
369
+ cache_read_tokens=int(row["cache_read_tokens"]),
370
+ cache_creation_tokens=int(row["cache_creation_tokens"]),
371
+ )
372
+
373
+
374
+ class PostgresRunStore:
375
+ """asyncpg-backed subagent run lifecycle store."""
376
+
377
+ def __init__(self, pool: asyncpg.Pool) -> None:
378
+ self._pool = pool
379
+
380
+ async def record_pending(
381
+ self,
382
+ run_id: str,
383
+ parent_run_id: str | None,
384
+ script: str,
385
+ envelope: SubagentEnvelope,
386
+ scratch_dir: object,
387
+ ) -> None:
388
+ await self._record_run("pending", run_id, parent_run_id, script, envelope, scratch_dir)
389
+
390
+ async def record_started(
391
+ self,
392
+ run_id: str,
393
+ parent_run_id: str | None,
394
+ script: str,
395
+ envelope: SubagentEnvelope,
396
+ scratch_dir: object,
397
+ ) -> None:
398
+ await self._record_run("running", run_id, parent_run_id, script, envelope, scratch_dir)
399
+
400
+ async def _record_run(
401
+ self,
402
+ status: str,
403
+ run_id: str,
404
+ parent_run_id: str | None,
405
+ script: str,
406
+ envelope: SubagentEnvelope,
407
+ scratch_dir: object,
408
+ ) -> None:
409
+ now_ms = int(time.time() * 1000)
410
+ async with self._pool.acquire() as conn:
411
+ await conn.execute(
412
+ """
413
+ INSERT INTO subagent_runs(
414
+ run_id, parent_run_id, script, envelope_json,
415
+ status, result_json, error_json, started_at, finished_at, scratch_dir,
416
+ worker_id, claimed_at
417
+ )
418
+ VALUES ($1, $2, $3, $4, $5, NULL, NULL, $6, NULL, $7, NULL, NULL)
419
+ """,
420
+ run_id,
421
+ parent_run_id,
422
+ script,
423
+ envelope.to_json(),
424
+ status,
425
+ now_ms,
426
+ str(scratch_dir),
427
+ )
428
+
429
+ async def claim(self, run_id: str, worker_id: str) -> bool:
430
+ now_ms = int(time.time() * 1000)
431
+ async with self._pool.acquire() as conn:
432
+ row = await conn.fetchrow(
433
+ """
434
+ UPDATE subagent_runs
435
+ SET status = 'running',
436
+ worker_id = $2,
437
+ claimed_at = $3
438
+ WHERE run_id = $1 AND status = 'pending'
439
+ RETURNING run_id
440
+ """,
441
+ run_id,
442
+ worker_id,
443
+ now_ms,
444
+ )
445
+ return row is not None
446
+
447
+ async def reset_stale_claims(self, stale_after_ms: int) -> int:
448
+ cutoff = int(time.time() * 1000) - stale_after_ms
449
+ async with self._pool.acquire() as conn:
450
+ result = await conn.execute(
451
+ """
452
+ UPDATE subagent_runs
453
+ SET status = 'pending',
454
+ worker_id = NULL,
455
+ claimed_at = NULL
456
+ WHERE status = 'running'
457
+ AND claimed_at IS NOT NULL
458
+ AND claimed_at < $1
459
+ """,
460
+ cutoff,
461
+ )
462
+ try:
463
+ return int(result.split()[-1])
464
+ except (ValueError, IndexError):
465
+ return 0
466
+
467
+ async def record_completed(self, run_id: str, result_json: str) -> None:
468
+ now_ms = int(time.time() * 1000)
469
+ async with self._pool.acquire() as conn:
470
+ await conn.execute(
471
+ """
472
+ UPDATE subagent_runs
473
+ SET status = 'completed',
474
+ result_json = $1,
475
+ finished_at = $2,
476
+ error_json = NULL
477
+ WHERE run_id = $3
478
+ """,
479
+ result_json,
480
+ now_ms,
481
+ run_id,
482
+ )
483
+
484
+ async def record_failed(self, run_id: str, error: str) -> None:
485
+ now_ms = int(time.time() * 1000)
486
+ err_payload = json.dumps({"message": error})
487
+ async with self._pool.acquire() as conn:
488
+ await conn.execute(
489
+ """
490
+ UPDATE subagent_runs
491
+ SET status = 'failed',
492
+ error_json = $1,
493
+ finished_at = $2
494
+ WHERE run_id = $3
495
+ """,
496
+ err_payload,
497
+ now_ms,
498
+ run_id,
499
+ )
500
+
501
+ async def pending_runs(self) -> list[SubagentRunRow]:
502
+ columns = ", ".join(_SUBAGENT_COLUMNS)
503
+ async with self._pool.acquire() as conn:
504
+ rows = await conn.fetch(
505
+ f"""
506
+ SELECT {columns} FROM subagent_runs
507
+ WHERE status = 'pending'
508
+ ORDER BY started_at ASC
509
+ """
510
+ )
511
+ return [_tuple_to_run_row(tuple(row)) for row in rows]
512
+
513
+ async def get_run(self, run_id: str) -> SubagentRunRow | None:
514
+ columns = ", ".join(_SUBAGENT_COLUMNS)
515
+ async with self._pool.acquire() as conn:
516
+ row = await conn.fetchrow(
517
+ f"SELECT {columns} FROM subagent_runs WHERE run_id = $1",
518
+ run_id,
519
+ )
520
+ if row is None:
521
+ return None
522
+ return _tuple_to_run_row(tuple(row))
523
+
524
+
525
+ class PostgresScheduledLoopStore:
526
+ """Postgres persistence for scheduled agent loops."""
527
+
528
+ def __init__(self, pool: asyncpg.Pool) -> None:
529
+ self._pool = pool
530
+
531
+ async def create(self, spec: ScheduledLoopCreate) -> ScheduledLoopRow:
532
+ loop_id = _loop_id_from_create(spec)
533
+ if await self.get(loop_id) is not None:
534
+ raise ValueError(f"scheduled loop already exists: {loop_id}")
535
+ validate_loop_guards(
536
+ max_ticks=spec.max_ticks,
537
+ max_runtime_ms=spec.max_runtime_ms,
538
+ unbounded=spec.unbounded,
539
+ )
540
+ now_ms = int(time.time() * 1000)
541
+ async with self._pool.acquire() as conn:
542
+ await conn.execute(
543
+ """
544
+ INSERT INTO scheduled_loops(
545
+ loop_id, session_id, status, prompt, interval_ms,
546
+ max_ticks, max_runtime_ms, skip_if_busy, tick_index,
547
+ next_tick_at_ms, started_at_ms, last_tick_at_ms,
548
+ last_error, stop_reason, tick_in_flight, worker_id, claimed_at_ms
549
+ ) VALUES ($1, $2, 'active', $3, $4, $5, $6, $7, 0, $8, $8, NULL, NULL, NULL, 0, NULL, NULL)
550
+ """,
551
+ loop_id,
552
+ spec.session_id.strip() or "loop-main",
553
+ spec.prompt.strip(),
554
+ spec.interval_ms,
555
+ spec.max_ticks,
556
+ spec.max_runtime_ms,
557
+ 1 if spec.skip_if_busy else 0,
558
+ now_ms,
559
+ )
560
+ row = await self.get(loop_id)
561
+ if row is None:
562
+ raise RuntimeError("failed to read scheduled loop after insert")
563
+ return row
564
+
565
+ async def get(self, loop_id: str) -> ScheduledLoopRow | None:
566
+ columns = ", ".join(_SCHEDULED_LOOP_COLUMNS)
567
+ async with self._pool.acquire() as conn:
568
+ row = await conn.fetchrow(
569
+ f"SELECT {columns} FROM scheduled_loops WHERE loop_id = $1",
570
+ loop_id,
571
+ )
572
+ if row is None:
573
+ return None
574
+ return _row_from_tuple(tuple(row))
575
+
576
+ async def list_all(self) -> list[ScheduledLoopRow]:
577
+ columns = ", ".join(_SCHEDULED_LOOP_COLUMNS)
578
+ async with self._pool.acquire() as conn:
579
+ rows = await conn.fetch(
580
+ f"SELECT {columns} FROM scheduled_loops ORDER BY started_at_ms DESC"
581
+ )
582
+ return [_row_from_tuple(tuple(r)) for r in rows]
583
+
584
+ async def list_due(self, now_ms: int) -> list[ScheduledLoopRow]:
585
+ columns = ", ".join(_SCHEDULED_LOOP_COLUMNS)
586
+ async with self._pool.acquire() as conn:
587
+ rows = await conn.fetch(
588
+ f"""
589
+ SELECT {columns} FROM scheduled_loops
590
+ WHERE status = 'active'
591
+ AND tick_in_flight = 0
592
+ AND next_tick_at_ms <= $1
593
+ ORDER BY next_tick_at_ms ASC
594
+ """,
595
+ now_ms,
596
+ )
597
+ return [_row_from_tuple(tuple(r)) for r in rows]
598
+
599
+ async def claim_tick(self, loop_id: str, worker_id: str) -> ScheduledLoopRow | None:
600
+ now_ms = int(time.time() * 1000)
601
+ async with self._pool.acquire() as conn:
602
+ status = await conn.execute(
603
+ """
604
+ UPDATE scheduled_loops
605
+ SET tick_in_flight = 1, worker_id = $1, claimed_at_ms = $2
606
+ WHERE loop_id = $3
607
+ AND status = 'active'
608
+ AND tick_in_flight = 0
609
+ AND next_tick_at_ms <= $2
610
+ """,
611
+ worker_id,
612
+ now_ms,
613
+ loop_id,
614
+ )
615
+ if status.split()[-1] != "1":
616
+ return None
617
+ return await self.get(loop_id)
618
+
619
+ async def release_stale_claims(self, stale_after_ms: int) -> int:
620
+ cutoff = int(time.time() * 1000) - stale_after_ms
621
+ async with self._pool.acquire() as conn:
622
+ status = await conn.execute(
623
+ """
624
+ UPDATE scheduled_loops
625
+ SET tick_in_flight = 0, worker_id = NULL, claimed_at_ms = NULL,
626
+ last_error = COALESCE(last_error, 'stale tick claim released')
627
+ WHERE tick_in_flight = 1
628
+ AND claimed_at_ms IS NOT NULL
629
+ AND claimed_at_ms < $1
630
+ """,
631
+ cutoff,
632
+ )
633
+ return int(status.split()[-1])
634
+
635
+ async def complete_tick(
636
+ self,
637
+ loop_id: str,
638
+ *,
639
+ worker_id: str,
640
+ error: str | None = None,
641
+ ) -> ScheduledLoopRow | None:
642
+ row = await self.get(loop_id)
643
+ if row is None or not row.tick_in_flight or row.worker_id != worker_id:
644
+ return None
645
+ now_ms = int(time.time() * 1000)
646
+ tick_index = row.tick_index + 1
647
+ stop_reason: str | None = None
648
+ status = row.status
649
+ if error:
650
+ status = "failed"
651
+ stop_reason = "tick_error"
652
+ elif row.max_ticks is not None and tick_index >= row.max_ticks:
653
+ status = "completed"
654
+ stop_reason = "max_ticks"
655
+ elif row.max_runtime_ms is not None and (now_ms - row.started_at_ms) >= row.max_runtime_ms:
656
+ status = "completed"
657
+ stop_reason = "max_runtime"
658
+ next_tick = now_ms + row.interval_ms if status == "active" else row.next_tick_at_ms
659
+ async with self._pool.acquire() as conn:
660
+ await conn.execute(
661
+ """
662
+ UPDATE scheduled_loops
663
+ SET tick_index = $1, last_tick_at_ms = $2, last_error = $3,
664
+ status = $4, stop_reason = $5, next_tick_at_ms = $6,
665
+ tick_in_flight = 0, worker_id = NULL, claimed_at_ms = NULL
666
+ WHERE loop_id = $7 AND worker_id = $8 AND tick_in_flight = 1
667
+ """,
668
+ tick_index,
669
+ now_ms,
670
+ error,
671
+ status,
672
+ stop_reason,
673
+ next_tick,
674
+ loop_id,
675
+ worker_id,
676
+ )
677
+ return await self.get(loop_id)
678
+
679
+ async def defer_tick(self, loop_id: str, *, worker_id: str, reason: str) -> None:
680
+ row = await self.get(loop_id)
681
+ if row is None or row.worker_id != worker_id:
682
+ return
683
+ now_ms = int(time.time() * 1000)
684
+ async with self._pool.acquire() as conn:
685
+ await conn.execute(
686
+ """
687
+ UPDATE scheduled_loops
688
+ SET tick_in_flight = 0, worker_id = NULL, claimed_at_ms = NULL,
689
+ next_tick_at_ms = $1, last_error = $2
690
+ WHERE loop_id = $3 AND worker_id = $4
691
+ """,
692
+ now_ms + row.interval_ms,
693
+ reason,
694
+ loop_id,
695
+ worker_id,
696
+ )
697
+
698
+ async def renew_tick_claim(self, loop_id: str, worker_id: str) -> bool:
699
+ now_ms = int(time.time() * 1000)
700
+ async with self._pool.acquire() as conn:
701
+ status = await conn.execute(
702
+ """
703
+ UPDATE scheduled_loops
704
+ SET claimed_at_ms = $1
705
+ WHERE loop_id = $2
706
+ AND worker_id = $3
707
+ AND tick_in_flight = 1
708
+ """,
709
+ now_ms,
710
+ loop_id,
711
+ worker_id,
712
+ )
713
+ return bool(status.split()[-1] == "1")
714
+
715
+ async def pause(self, loop_id: str) -> bool:
716
+ async with self._pool.acquire() as conn:
717
+ status = await conn.execute(
718
+ """
719
+ UPDATE scheduled_loops
720
+ SET status = 'paused', tick_in_flight = 0, worker_id = NULL, claimed_at_ms = NULL
721
+ WHERE loop_id = $1
722
+ """,
723
+ loop_id,
724
+ )
725
+ return bool(status.split()[-1] == "1")
726
+
727
+ async def resume(self, loop_id: str) -> bool:
728
+ now_ms = int(time.time() * 1000)
729
+ async with self._pool.acquire() as conn:
730
+ status = await conn.execute(
731
+ """
732
+ UPDATE scheduled_loops
733
+ SET status = 'active', stop_reason = NULL, next_tick_at_ms = $1,
734
+ tick_in_flight = 0, worker_id = NULL, claimed_at_ms = NULL
735
+ WHERE loop_id = $2 AND status = 'paused'
736
+ """,
737
+ now_ms,
738
+ loop_id,
739
+ )
740
+ return bool(status.split()[-1] == "1")
741
+
742
+ async def stop(self, loop_id: str, *, stop_reason: str = "manual") -> bool:
743
+ async with self._pool.acquire() as conn:
744
+ status = await conn.execute(
745
+ """
746
+ UPDATE scheduled_loops
747
+ SET status = 'completed', stop_reason = $1,
748
+ tick_in_flight = 0, worker_id = NULL, claimed_at_ms = NULL
749
+ WHERE loop_id = $2
750
+ """,
751
+ stop_reason,
752
+ loop_id,
753
+ )
754
+ return bool(status.split()[-1] == "1")
755
+
756
+
757
+ class PostgresSessionTurnLockStore:
758
+ """Postgres-backed exclusive turn lock per session."""
759
+
760
+ def __init__(self, pool: asyncpg.Pool) -> None:
761
+ self._pool = pool
762
+
763
+ async def release_stale_claims(self, stale_after_ms: int) -> int:
764
+ cutoff = int(time.time() * 1000) - stale_after_ms
765
+ async with self._pool.acquire() as conn:
766
+ status = await conn.execute(
767
+ """
768
+ UPDATE session_turn_locks
769
+ SET request_id = NULL, claimed_at_ms = NULL
770
+ WHERE request_id IS NOT NULL
771
+ AND claimed_at_ms IS NOT NULL
772
+ AND claimed_at_ms < $1
773
+ """,
774
+ cutoff,
775
+ )
776
+ return int(status.split()[-1])
777
+
778
+ async def try_acquire(self, session_id: str, request_id: str) -> bool:
779
+ from monkeybot.core.persistence.session_turn_locks import session_turn_stale_ms
780
+
781
+ await self.release_stale_claims(session_turn_stale_ms())
782
+ now_ms = int(time.time() * 1000)
783
+ async with self._pool.acquire() as conn:
784
+ row = await conn.fetchrow(
785
+ """
786
+ UPDATE session_turn_locks
787
+ SET request_id = $2, claimed_at_ms = $3
788
+ WHERE session_id = $1 AND request_id IS NULL
789
+ RETURNING session_id
790
+ """,
791
+ session_id,
792
+ request_id,
793
+ now_ms,
794
+ )
795
+ if row is not None:
796
+ return True
797
+ try:
798
+ await conn.execute(
799
+ """
800
+ INSERT INTO session_turn_locks (session_id, request_id, claimed_at_ms)
801
+ VALUES ($1, $2, $3)
802
+ """,
803
+ session_id,
804
+ request_id,
805
+ now_ms,
806
+ )
807
+ return True
808
+ except asyncpg.UniqueViolationError:
809
+ return False
810
+
811
+ async def release(self, session_id: str, request_id: str) -> None:
812
+ async with self._pool.acquire() as conn:
813
+ await conn.execute(
814
+ """
815
+ UPDATE session_turn_locks
816
+ SET request_id = NULL, claimed_at_ms = NULL
817
+ WHERE session_id = $1 AND request_id = $2
818
+ """,
819
+ session_id,
820
+ request_id,
821
+ )
822
+
823
+ async def is_busy(self, session_id: str) -> bool:
824
+ async with self._pool.acquire() as conn:
825
+ row = await conn.fetchrow(
826
+ """
827
+ SELECT 1 FROM session_turn_locks
828
+ WHERE session_id = $1 AND request_id IS NOT NULL
829
+ LIMIT 1
830
+ """,
831
+ session_id,
832
+ )
833
+ return row is not None
834
+
835
+
836
+ class PostgresStorageBackend:
837
+ """Postgres-backed storage backend using an asyncpg connection pool."""
838
+
839
+ def __init__(self, db_url: str) -> None:
840
+ self._db_url = db_url
841
+ self._pool: asyncpg.Pool | None = None
842
+ self._history_store: PostgresHistoryStore | None = None
843
+ self._usage_store: PostgresUsageStore | None = None
844
+ self._runs_store: PostgresRunStore | None = None
845
+ self._scheduled_loops_store: PostgresScheduledLoopStore | None = None
846
+ self._session_turn_lock_store: PostgresSessionTurnLockStore | None = None
847
+
848
+ async def open(self, *, run_schema: bool = True) -> None:
849
+ min_size = int(os.environ.get("POSTGRES_POOL_MIN", "1"))
850
+ max_size = int(os.environ.get("POSTGRES_POOL_MAX", "5"))
851
+ self._pool = await asyncpg.create_pool(
852
+ self._db_url, min_size=min_size, max_size=max_size
853
+ )
854
+ if run_schema:
855
+ await _apply_schema(self._pool)
856
+ self._history_store = PostgresHistoryStore(self._pool)
857
+ self._usage_store = PostgresUsageStore(self._pool)
858
+ self._runs_store = PostgresRunStore(self._pool)
859
+ self._scheduled_loops_store = PostgresScheduledLoopStore(self._pool)
860
+ self._session_turn_lock_store = PostgresSessionTurnLockStore(self._pool)
861
+
862
+ async def close(self) -> None:
863
+ if self._pool is not None:
864
+ await self._pool.close()
865
+ self._pool = None
866
+ self._history_store = None
867
+ self._usage_store = None
868
+ self._runs_store = None
869
+ self._scheduled_loops_store = None
870
+ self._session_turn_lock_store = None
871
+
872
+ def history(self) -> PostgresHistoryStore:
873
+ if self._history_store is None:
874
+ raise RuntimeError("PostgresStorageBackend.open() has not been called")
875
+ return self._history_store
876
+
877
+ def usage(self) -> PostgresUsageStore:
878
+ if self._usage_store is None:
879
+ raise RuntimeError("PostgresStorageBackend.open() has not been called")
880
+ return self._usage_store
881
+
882
+ def runs(self) -> PostgresRunStore:
883
+ if self._runs_store is None:
884
+ raise RuntimeError("PostgresStorageBackend.open() has not been called")
885
+ return self._runs_store
886
+
887
+ def scheduled_loops(self) -> PostgresScheduledLoopStore:
888
+ if self._scheduled_loops_store is None:
889
+ raise RuntimeError("PostgresStorageBackend.open() has not been called")
890
+ return self._scheduled_loops_store
891
+
892
+ def session_turns(self) -> PostgresSessionTurnLockStore:
893
+ if self._session_turn_lock_store is None:
894
+ raise RuntimeError("PostgresStorageBackend.open() has not been called")
895
+ return self._session_turn_lock_store