monkeybot 2.1.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (178) hide show
  1. monkeybot/__init__.py +3 -0
  2. monkeybot/cli/__init__.py +3 -0
  3. monkeybot/cli/__main__.py +8 -0
  4. monkeybot/cli/audio_io.py +8 -0
  5. monkeybot/cli/gateway_manager.py +17 -0
  6. monkeybot/cli/main.py +22 -0
  7. monkeybot/cli/push_to_talk.py +12 -0
  8. monkeybot/cli/realtime_client.py +13 -0
  9. monkeybot/core/__init__.py +19 -0
  10. monkeybot/core/attachments/__init__.py +22 -0
  11. monkeybot/core/attachments/catalog.py +62 -0
  12. monkeybot/core/attachments/config.py +52 -0
  13. monkeybot/core/attachments/freeze.py +158 -0
  14. monkeybot/core/attachments/resolve.py +70 -0
  15. monkeybot/core/attachments/store.py +180 -0
  16. monkeybot/core/attachments/text.py +72 -0
  17. monkeybot/core/attachments/tools.py +54 -0
  18. monkeybot/core/bootstrap.py +242 -0
  19. monkeybot/core/config/__init__.py +71 -0
  20. monkeybot/core/config/realtime_config.py +150 -0
  21. monkeybot/core/config/runtime_env.py +262 -0
  22. monkeybot/core/config/settings.py +341 -0
  23. monkeybot/core/config/validation.py +249 -0
  24. monkeybot/core/config/yaml_loader.py +45 -0
  25. monkeybot/core/context/__init__.py +781 -0
  26. monkeybot/core/context/campaign_context.py +8 -0
  27. monkeybot/core/context/common.py +14 -0
  28. monkeybot/core/context/curator.py +255 -0
  29. monkeybot/core/context/epoch.py +226 -0
  30. monkeybot/core/context/memory_prompt.py +222 -0
  31. monkeybot/core/context/tool_output_policy.py +270 -0
  32. monkeybot/core/context/tool_result_ingress.py +290 -0
  33. monkeybot/core/context/tool_shapers.py +361 -0
  34. monkeybot/core/hooks/__init__.py +261 -0
  35. monkeybot/core/llm/__init__.py +4 -0
  36. monkeybot/core/llm/provider.py +296 -0
  37. monkeybot/core/llm/realtime_provider.py +203 -0
  38. monkeybot/core/llm/usage.py +57 -0
  39. monkeybot/core/logging_utils.py +24 -0
  40. monkeybot/core/mcp/__init__.py +1 -0
  41. monkeybot/core/mcp/mcp_client.py +1215 -0
  42. monkeybot/core/mcp/ports_mcp.py +109 -0
  43. monkeybot/core/memory/__init__.py +24 -0
  44. monkeybot/core/memory/hook.py +413 -0
  45. monkeybot/core/memory/index_format.py +104 -0
  46. monkeybot/core/memory/integrity.py +180 -0
  47. monkeybot/core/memory/organizer.py +270 -0
  48. monkeybot/core/memory/storage_ops.py +139 -0
  49. monkeybot/core/memory/subsystem.py +91 -0
  50. monkeybot/core/messages/__init__.py +16 -0
  51. monkeybot/core/messages/convert_provider.py +41 -0
  52. monkeybot/core/messages/tool_integrity.py +262 -0
  53. monkeybot/core/messages/transform_context.py +84 -0
  54. monkeybot/core/path_safety.py +11 -0
  55. monkeybot/core/persistence/__init__.py +17 -0
  56. monkeybot/core/persistence/backends.py +236 -0
  57. monkeybot/core/persistence/db.py +28 -0
  58. monkeybot/core/persistence/durable_runs.py +286 -0
  59. monkeybot/core/persistence/firestore.py +658 -0
  60. monkeybot/core/persistence/firestore_scheduled_loops.py +336 -0
  61. monkeybot/core/persistence/history.py +156 -0
  62. monkeybot/core/persistence/postgres.py +895 -0
  63. monkeybot/core/persistence/runs.py +76 -0
  64. monkeybot/core/persistence/scheduled_loops.py +435 -0
  65. monkeybot/core/persistence/session_turn_locks.py +94 -0
  66. monkeybot/core/persistence/sqlite.py +218 -0
  67. monkeybot/core/persistence/sqlite_backend.py +74 -0
  68. monkeybot/core/persistence/thread_summary.py +61 -0
  69. monkeybot/core/persistence/transcript.py +194 -0
  70. monkeybot/core/persistence/usage.py +149 -0
  71. monkeybot/core/prompts/__init__.py +1 -0
  72. monkeybot/core/prompts/harness_prompt.py +197 -0
  73. monkeybot/core/prompts/prompt.py +215 -0
  74. monkeybot/core/runtime/__init__.py +1 -0
  75. monkeybot/core/runtime/context_budget.py +267 -0
  76. monkeybot/core/runtime/events.py +819 -0
  77. monkeybot/core/runtime/input_admission.py +154 -0
  78. monkeybot/core/runtime/loop.py +2374 -0
  79. monkeybot/core/runtime/provider_stream_mapper.py +159 -0
  80. monkeybot/core/runtime/realtime_loop.py +654 -0
  81. monkeybot/core/runtime/utterance_buffer.py +179 -0
  82. monkeybot/core/subagents/__init__.py +1 -0
  83. monkeybot/core/subagents/subagent_proto.py +331 -0
  84. monkeybot/core/subagents/subagent_worker.py +441 -0
  85. monkeybot/core/subagents/worker_pool.py +403 -0
  86. monkeybot/core/testing/__init__.py +1 -0
  87. monkeybot/core/testing/mocks_provider.py +86 -0
  88. monkeybot/core/testing/mocks_realtime_provider.py +137 -0
  89. monkeybot/core/tools/__init__.py +1 -0
  90. monkeybot/core/tools/core_tool_executor.py +1548 -0
  91. monkeybot/core/tools/inspector.py +226 -0
  92. monkeybot/core/tools/loop_inspector.py +45 -0
  93. monkeybot/core/tools/patch.py +480 -0
  94. monkeybot/core/tools/permission.py +284 -0
  95. monkeybot/core/tools/sandbox_executor.py +255 -0
  96. monkeybot/core/tools/spill_inventory.py +35 -0
  97. monkeybot/core/tools/terminal.py +381 -0
  98. monkeybot/core/tools/text_normalize.py +25 -0
  99. monkeybot/core/tools/types.py +33 -0
  100. monkeybot/core/tools/workspace_service.py +710 -0
  101. monkeybot/core/tools/workspace_tools.py +116 -0
  102. monkeybot/core/types/__init__.py +1 -0
  103. monkeybot/core/types/content_blocks.py +644 -0
  104. monkeybot/core/types/interfaces.py +156 -0
  105. monkeybot/core/types/types_tools.py +29 -0
  106. monkeybot/core/workspace/__init__.py +8 -0
  107. monkeybot/core/workspace/factory.py +45 -0
  108. monkeybot/core/workspace/gcs.py +130 -0
  109. monkeybot/core/workspace/local.py +162 -0
  110. monkeybot/core/workspace/protocol.py +45 -0
  111. monkeybot/core/workspace/s3.py +151 -0
  112. monkeybot/core/workspace_layout.py +27 -0
  113. monkeybot/gateway/__init__.py +1 -0
  114. monkeybot/gateway/bootstrap.py +18 -0
  115. monkeybot/gateway/main.py +47 -0
  116. monkeybot/gateway/realtime/__init__.py +31 -0
  117. monkeybot/gateway/realtime/app.py +321 -0
  118. monkeybot/gateway/realtime/deps.py +52 -0
  119. monkeybot/gateway/realtime/errors.py +81 -0
  120. monkeybot/gateway/realtime/guardrails.py +88 -0
  121. monkeybot/gateway/realtime/manager.py +77 -0
  122. monkeybot/gateway/realtime/metrics.py +144 -0
  123. monkeybot/gateway/realtime/routes.py +864 -0
  124. monkeybot/gateway/realtime/session.py +232 -0
  125. monkeybot/gateway/realtime/wire.py +412 -0
  126. monkeybot/gateway/realtime_main.py +49 -0
  127. monkeybot/gateway/sse/__init__.py +1 -0
  128. monkeybot/gateway/sse/app.py +733 -0
  129. monkeybot/gateway/sse/loop_port.py +31 -0
  130. monkeybot/gateway/sse/models.py +177 -0
  131. monkeybot/gateway/sse/reply_body.py +91 -0
  132. monkeybot/gateway/sse/routes.py +1101 -0
  133. monkeybot/gateway/sse/scheduler_routes.py +200 -0
  134. monkeybot/gateway/sse/scheduler_wiring.py +96 -0
  135. monkeybot/gateway/sse/session_bus.py +226 -0
  136. monkeybot/gateway/sse/sse.py +46 -0
  137. monkeybot/gateway/sse/workspace_layout.py +7 -0
  138. monkeybot/observability/__init__.py +220 -0
  139. monkeybot/observability/_state.py +10 -0
  140. monkeybot/observability/instrumentation.py +153 -0
  141. monkeybot/observability/propagation.py +65 -0
  142. monkeybot/observability/spans.py +455 -0
  143. monkeybot/providers/__init__.py +19 -0
  144. monkeybot/providers/_openai_compat.py +450 -0
  145. monkeybot/providers/_utils.py +473 -0
  146. monkeybot/providers/bedrock.py +145 -0
  147. monkeybot/providers/claude.py +125 -0
  148. monkeybot/providers/gemini.py +677 -0
  149. monkeybot/providers/gemini_live.py +398 -0
  150. monkeybot/providers/huggingface.py +129 -0
  151. monkeybot/providers/nvidia.py +104 -0
  152. monkeybot/providers/ollama.py +152 -0
  153. monkeybot/providers/openai.py +127 -0
  154. monkeybot/providers/pricing.py +60 -0
  155. monkeybot/providers/sampling.py +44 -0
  156. monkeybot/providers/vertex_claude.py +148 -0
  157. monkeybot/scaffold/__init__.py +33 -0
  158. monkeybot/scheduler/__init__.py +13 -0
  159. monkeybot/scheduler/__main__.py +4 -0
  160. monkeybot/scheduler/engine.py +333 -0
  161. monkeybot/scheduler/http_invoker.py +61 -0
  162. monkeybot/scheduler/interval.py +77 -0
  163. monkeybot/scheduler/tick_result.py +34 -0
  164. monkeybot/scheduler/worker.py +87 -0
  165. monkeybot/subagents/__init__.py +1 -0
  166. monkeybot/subagents/worker/__init__.py +1 -0
  167. monkeybot/subagents/worker/__main__.py +22 -0
  168. monkeybot/web_search/__init__.py +82 -0
  169. monkeybot/web_search/backends/__init__.py +5 -0
  170. monkeybot/web_search/backends/duckduckgo.py +32 -0
  171. monkeybot/web_search/backends/firecrawl.py +43 -0
  172. monkeybot/web_search/backends/tavily.py +45 -0
  173. monkeybot/web_search/protocol.py +25 -0
  174. monkeybot/web_search/tool.py +56 -0
  175. monkeybot-2.1.1.dist-info/METADATA +318 -0
  176. monkeybot-2.1.1.dist-info/RECORD +178 -0
  177. monkeybot-2.1.1.dist-info/WHEEL +4 -0
  178. monkeybot-2.1.1.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,218 @@
1
+ """SQLite connection helpers and schema DDL for monkeybot persistence."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import logging
6
+ import os
7
+ from pathlib import Path
8
+ from typing import Final
9
+
10
+ import aiosqlite
11
+
12
+ logger = logging.getLogger(__name__)
13
+
14
+ DEFAULT_DB_URL: Final[str] = "sqlite:///data/monkeybot.db"
15
+
16
+ _LEGACY_SCHEMA_MESSAGE = """Legacy conversation_history schema detected (tool_name and/or
17
+ tool_call_id columns present). The on-disk format changed in the
18
+ typed-message-content release. Choose one:
19
+
20
+ Dev / demo agent:
21
+ rm -f demo_agent/workspace/data/monkeybot.db
22
+ rm -f demo_agent/workspace/data/monkeybot.db-shm
23
+ rm -f demo_agent/workspace/data/monkeybot.db-wal
24
+
25
+ Other deployments: there is no automated migration. The legacy format is
26
+ a JSON tail inside a TEXT column; transcribing it to typed blocks must
27
+ be done by the operator. See docs/migrations/typed-message-content.md."""
28
+
29
+ SCHEMA_DDLS: Final[tuple[str, ...]] = (
30
+ """CREATE TABLE IF NOT EXISTS conversation_history (
31
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
32
+ thread_id TEXT NOT NULL,
33
+ role TEXT NOT NULL,
34
+ content TEXT NOT NULL,
35
+ created_at INTEGER NOT NULL
36
+ )""",
37
+ """CREATE TABLE IF NOT EXISTS subagent_runs (
38
+ run_id TEXT PRIMARY KEY,
39
+ parent_run_id TEXT,
40
+ script TEXT NOT NULL,
41
+ envelope_json TEXT NOT NULL,
42
+ status TEXT NOT NULL,
43
+ result_json TEXT,
44
+ error_json TEXT,
45
+ started_at INTEGER,
46
+ finished_at INTEGER,
47
+ scratch_dir TEXT,
48
+ worker_id TEXT,
49
+ claimed_at INTEGER
50
+ )""",
51
+ """CREATE TABLE IF NOT EXISTS turn_usage (
52
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
53
+ thread_id TEXT NOT NULL,
54
+ run_id TEXT,
55
+ model TEXT NOT NULL,
56
+ input_tokens INTEGER NOT NULL,
57
+ output_tokens INTEGER NOT NULL,
58
+ cached_tokens INTEGER NOT NULL,
59
+ cost_usd REAL NOT NULL,
60
+ duration_ms INTEGER NOT NULL,
61
+ created_at INTEGER NOT NULL,
62
+ context_json TEXT,
63
+ estimated_prompt_tokens INTEGER NOT NULL DEFAULT 0,
64
+ cache_read_tokens INTEGER NOT NULL DEFAULT 0,
65
+ cache_creation_tokens INTEGER NOT NULL DEFAULT 0
66
+ )""",
67
+ "CREATE INDEX IF NOT EXISTS idx_history_thread ON conversation_history(thread_id, created_at)",
68
+ """CREATE INDEX IF NOT EXISTS idx_history_thread_last
69
+ ON conversation_history(thread_id, created_at DESC, id DESC)""",
70
+ "CREATE INDEX IF NOT EXISTS idx_runs_parent ON subagent_runs(parent_run_id)",
71
+ """CREATE INDEX IF NOT EXISTS idx_runs_status ON subagent_runs(status)
72
+ WHERE status IN ('pending','running')""",
73
+ "CREATE INDEX IF NOT EXISTS idx_usage_thread ON turn_usage(thread_id, created_at)",
74
+ "CREATE INDEX IF NOT EXISTS idx_usage_cost ON turn_usage(created_at)",
75
+ """CREATE TABLE IF NOT EXISTS scheduled_loops (
76
+ loop_id TEXT PRIMARY KEY,
77
+ session_id TEXT NOT NULL,
78
+ status TEXT NOT NULL,
79
+ prompt TEXT NOT NULL,
80
+ interval_ms INTEGER NOT NULL,
81
+ max_ticks INTEGER,
82
+ max_runtime_ms INTEGER,
83
+ skip_if_busy INTEGER NOT NULL DEFAULT 1,
84
+ tick_index INTEGER NOT NULL DEFAULT 0,
85
+ next_tick_at_ms INTEGER NOT NULL,
86
+ started_at_ms INTEGER NOT NULL,
87
+ last_tick_at_ms INTEGER,
88
+ last_error TEXT,
89
+ stop_reason TEXT,
90
+ tick_in_flight INTEGER NOT NULL DEFAULT 0,
91
+ worker_id TEXT,
92
+ claimed_at_ms INTEGER
93
+ )""",
94
+ """CREATE INDEX IF NOT EXISTS idx_scheduled_loops_due
95
+ ON scheduled_loops(status, tick_in_flight, next_tick_at_ms)
96
+ WHERE status = 'active'""",
97
+ """CREATE TABLE IF NOT EXISTS session_turn_locks (
98
+ session_id TEXT PRIMARY KEY,
99
+ request_id TEXT,
100
+ claimed_at_ms INTEGER
101
+ )""",
102
+ )
103
+
104
+
105
+ def sqlite_path_from_db_url(db_url: str | None = None) -> str:
106
+ """Resolve sqlite database path suitable for aiosqlite.connect().
107
+
108
+ Accepts sqlite:////absolute/path, sqlite:///relative/path, sqlite:///:memory:.
109
+ If db_url is None, reads os.environ.get('DB_URL', DEFAULT_DB_URL).
110
+ Raises ValueError if scheme is not sqlite or path is empty.
111
+ """
112
+ if db_url is None:
113
+ db_url = os.environ.get("DB_URL", DEFAULT_DB_URL)
114
+ stripped = db_url.strip()
115
+ if not stripped:
116
+ raise ValueError("Database URL is empty")
117
+ prefix = "sqlite:///"
118
+ if not stripped.lower().startswith(prefix):
119
+ raise ValueError(f"Unsupported database URL: {db_url!r}")
120
+ remainder = stripped[len(prefix) :]
121
+ if not remainder:
122
+ raise ValueError("SQLite URL path is empty")
123
+ if remainder == ":memory:":
124
+ return ":memory:"
125
+ return remainder
126
+
127
+
128
+ async def configure_connection(conn: aiosqlite.Connection) -> None:
129
+ """PRAGMA journal_mode=WAL; foreign_keys=ON; synchronous=NORMAL."""
130
+ await conn.execute("PRAGMA journal_mode=WAL")
131
+ await conn.execute("PRAGMA foreign_keys=ON")
132
+ await conn.execute("PRAGMA synchronous=NORMAL")
133
+
134
+
135
+ async def _connection_db_path(conn: aiosqlite.Connection) -> str:
136
+ """Resolve main DB path for logging (``:memory:`` when no file path)."""
137
+ cursor = await conn.execute("PRAGMA database_list")
138
+ rows = await cursor.fetchall()
139
+ await cursor.close()
140
+ for row in rows:
141
+ if len(row) >= 3 and str(row[1]) == "main":
142
+ file_col = row[2]
143
+ return ":memory:" if not file_col else str(file_col)
144
+ return ":memory:"
145
+
146
+
147
+ async def _log_legacy_schema_error(conn: aiosqlite.Connection) -> None:
148
+ path = await _connection_db_path(conn)
149
+ logger.error("Legacy conversation_history schema detected; db=%s", path)
150
+
151
+
152
+ async def apply_schema(conn: aiosqlite.Connection) -> None:
153
+ """Create tables/indexes; refuse legacy conversation_history shapes."""
154
+ for ddl in SCHEMA_DDLS:
155
+ await conn.execute(ddl)
156
+ await conn.commit()
157
+ await _ensure_turn_usage_estimated_column(conn)
158
+ await _ensure_turn_usage_cache_columns(conn)
159
+ await _ensure_subagent_runs_claim_columns(conn)
160
+ cursor = await conn.execute("PRAGMA table_info(conversation_history)")
161
+ rows = await cursor.fetchall()
162
+ await cursor.close()
163
+ col_names = {str(r[1]) for r in rows}
164
+ if "tool_name" in col_names or "tool_call_id" in col_names:
165
+ await _log_legacy_schema_error(conn)
166
+ raise RuntimeError(_LEGACY_SCHEMA_MESSAGE)
167
+
168
+
169
+ async def _ensure_turn_usage_estimated_column(conn: aiosqlite.Connection) -> None:
170
+ """Add ``estimated_prompt_tokens`` when upgrading an existing DB."""
171
+ cur = await conn.execute("PRAGMA table_info(turn_usage)")
172
+ rows = await cur.fetchall()
173
+ await cur.close()
174
+ names = {str(r[1]) for r in rows}
175
+ if "estimated_prompt_tokens" in names:
176
+ return
177
+ await conn.execute(
178
+ "ALTER TABLE turn_usage ADD COLUMN estimated_prompt_tokens INTEGER NOT NULL DEFAULT 0"
179
+ )
180
+ await conn.commit()
181
+
182
+
183
+ async def _ensure_turn_usage_cache_columns(conn: aiosqlite.Connection) -> None:
184
+ """Add cache token columns when upgrading an existing DB."""
185
+ cur = await conn.execute("PRAGMA table_info(turn_usage)")
186
+ rows = await cur.fetchall()
187
+ await cur.close()
188
+ names = {str(r[1]) for r in rows}
189
+ for col in ("cache_read_tokens", "cache_creation_tokens"):
190
+ if col in names:
191
+ continue
192
+ await conn.execute(
193
+ f"ALTER TABLE turn_usage ADD COLUMN {col} INTEGER NOT NULL DEFAULT 0"
194
+ )
195
+ await conn.commit()
196
+
197
+
198
+ async def _ensure_subagent_runs_claim_columns(conn: aiosqlite.Connection) -> None:
199
+ """Add worker claim columns when upgrading an existing DB."""
200
+ cur = await conn.execute("PRAGMA table_info(subagent_runs)")
201
+ rows = await cur.fetchall()
202
+ await cur.close()
203
+ names = {str(r[1]) for r in rows}
204
+ if "worker_id" not in names:
205
+ await conn.execute("ALTER TABLE subagent_runs ADD COLUMN worker_id TEXT")
206
+ if "claimed_at" not in names:
207
+ await conn.execute("ALTER TABLE subagent_runs ADD COLUMN claimed_at INTEGER")
208
+ await conn.commit()
209
+
210
+
211
+ async def open_connection(db_url: str | None = None) -> aiosqlite.Connection:
212
+ """Open aiosqlite connection, configure WAL, return ready connection."""
213
+ path = sqlite_path_from_db_url(db_url)
214
+ if path != ":memory:":
215
+ Path(path).expanduser().resolve().parent.mkdir(parents=True, exist_ok=True)
216
+ conn = await aiosqlite.connect(path)
217
+ await configure_connection(conn)
218
+ return conn
@@ -0,0 +1,74 @@
1
+ """SQLite-backed StorageBackend implementation.
2
+
3
+ Lazy-imported by :func:`monkeybot.core.persistence.backends.create_storage_backend`
4
+ when a ``sqlite://`` URL is used — never loaded in Postgres-only deployments.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import aiosqlite
10
+
11
+ from monkeybot.core.persistence.durable_runs import SQLiteRunStore
12
+ from monkeybot.core.persistence.scheduled_loops import SQLiteScheduledLoopStore
13
+ from monkeybot.core.persistence.session_turn_locks import SQLiteSessionTurnLockStore
14
+ from monkeybot.core.persistence.history import SQLiteHistoryStore
15
+ from monkeybot.core.persistence.sqlite import apply_schema, open_connection
16
+ from monkeybot.core.persistence.usage import SQLiteUsageStore
17
+
18
+
19
+ class SQLiteStorageBackend:
20
+ """SQLite-backed storage backend. One shared connection for the process lifetime."""
21
+
22
+ def __init__(self, db_url: str) -> None:
23
+ self._db_url = db_url
24
+ self._conn: aiosqlite.Connection | None = None
25
+ self._history_store: SQLiteHistoryStore | None = None
26
+ self._usage_store: SQLiteUsageStore | None = None
27
+ self._runs_store: SQLiteRunStore | None = None
28
+ self._scheduled_loops_store: SQLiteScheduledLoopStore | None = None
29
+ self._session_turn_lock_store: SQLiteSessionTurnLockStore | None = None
30
+
31
+ async def open(self, *, run_schema: bool = True) -> None:
32
+ self._conn = await open_connection(self._db_url)
33
+ if run_schema:
34
+ await apply_schema(self._conn)
35
+ self._history_store = SQLiteHistoryStore(self._conn)
36
+ self._usage_store = SQLiteUsageStore(self._conn)
37
+ self._runs_store = SQLiteRunStore(self._conn)
38
+ self._scheduled_loops_store = SQLiteScheduledLoopStore(self._conn)
39
+ self._session_turn_lock_store = SQLiteSessionTurnLockStore(self._conn)
40
+
41
+ async def close(self) -> None:
42
+ if self._conn is not None:
43
+ await self._conn.close()
44
+ self._conn = None
45
+ self._history_store = None
46
+ self._usage_store = None
47
+ self._runs_store = None
48
+ self._scheduled_loops_store = None
49
+ self._session_turn_lock_store = None
50
+
51
+ def history(self) -> SQLiteHistoryStore:
52
+ if self._history_store is None:
53
+ raise RuntimeError("SQLiteStorageBackend.open() has not been called")
54
+ return self._history_store
55
+
56
+ def usage(self) -> SQLiteUsageStore:
57
+ if self._usage_store is None:
58
+ raise RuntimeError("SQLiteStorageBackend.open() has not been called")
59
+ return self._usage_store
60
+
61
+ def runs(self) -> SQLiteRunStore:
62
+ if self._runs_store is None:
63
+ raise RuntimeError("SQLiteStorageBackend.open() has not been called")
64
+ return self._runs_store
65
+
66
+ def scheduled_loops(self) -> SQLiteScheduledLoopStore:
67
+ if self._scheduled_loops_store is None:
68
+ raise RuntimeError("SQLiteStorageBackend.open() has not been called")
69
+ return self._scheduled_loops_store
70
+
71
+ def session_turns(self) -> SQLiteSessionTurnLockStore:
72
+ if self._session_turn_lock_store is None:
73
+ raise RuntimeError("SQLiteStorageBackend.open() has not been called")
74
+ return self._session_turn_lock_store
@@ -0,0 +1,61 @@
1
+ """Shared helpers for listing persisted chat threads."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ from dataclasses import dataclass
7
+
8
+ from monkeybot.core.llm.provider import Message
9
+ from monkeybot.core.types.content_blocks import Text
10
+
11
+
12
+ @dataclass(frozen=True)
13
+ class ChatThreadSummary:
14
+ """One row in a chat-thread picker."""
15
+
16
+ thread_id: str
17
+ last_message_at: int
18
+ message_count: int
19
+ preview: str
20
+
21
+
22
+ def preview_from_content_blob(content_blob: str, *, max_len: int = 120) -> str:
23
+ """Build a one-line preview from stored JSON content blocks."""
24
+ try:
25
+ raw = json.loads(content_blob)
26
+ except json.JSONDecodeError:
27
+ return ""
28
+ if not isinstance(raw, list):
29
+ return ""
30
+ parts: list[str] = []
31
+ for item in raw:
32
+ if isinstance(item, dict) and item.get("type") == "text":
33
+ text = item.get("text")
34
+ if isinstance(text, str) and text.strip():
35
+ parts.append(text.strip())
36
+ joined = " ".join(parts).replace("\n", " ").strip()
37
+ if len(joined) <= max_len:
38
+ return joined
39
+ return joined[: max_len - 1].rstrip() + "…"
40
+
41
+
42
+ def text_from_message(message: Message) -> str:
43
+ """Concatenate Text blocks from a stored message."""
44
+ parts: list[str] = []
45
+ for block in message.content:
46
+ if isinstance(block, Text) and block.text.strip():
47
+ parts.append(block.text.strip())
48
+ return "\n".join(parts).strip()
49
+
50
+
51
+ def messages_to_wire(messages: list[Message]) -> list[dict[str, str]]:
52
+ """Serialize user/assistant turns for the chat-history API."""
53
+ out: list[dict[str, str]] = []
54
+ for msg in messages:
55
+ if msg.role not in ("user", "assistant"):
56
+ continue
57
+ text = text_from_message(msg)
58
+ if not text:
59
+ continue
60
+ out.append({"role": msg.role, "text": text})
61
+ return out
@@ -0,0 +1,194 @@
1
+ """Session transcript capture for harness debugging (internal use only).
2
+
3
+ Writes one append-only NDJSON file per session under
4
+ ``{workspace_root}/.monkeybot/transcripts/{session_id}.ndjson``: a manifest
5
+ record on the first write, followed by durable :class:`~monkeybot.core.runtime.events.AgentEvent`
6
+ boundaries (OpenCode V2-style), plus raw provider request/response records.
7
+
8
+ Live-only streaming deltas (``AssistantDelta``, ``ToolInputDelta``, …) are
9
+ skipped by default so the transcript is a replay-grade durable log. Set
10
+ ``MONKEYBOT_TRANSCRIPT_INCLUDE_LIVE=1`` to capture every SSE event.
11
+
12
+ Not surfaced to the agent or any tool; gated by ``MONKEYBOT_TRANSCRIPT_ENABLED``
13
+ (default off) and wired only from the gateway loop (``GatewayLoopPort.start_turn``
14
+ and the provider call site in ``core/runtime/loop.py``).
15
+ """
16
+
17
+ from __future__ import annotations
18
+
19
+ import asyncio
20
+ import json
21
+ import logging
22
+ import os
23
+ import time
24
+ from pathlib import Path
25
+ from typing import Any
26
+
27
+ from monkeybot.core.path_safety import sanitize_path_component
28
+ from monkeybot.core.runtime.events import AgentEvent, event_to_json, is_durable_event
29
+
30
+ logger = logging.getLogger(__name__)
31
+
32
+ _TRANSCRIPT_REL_DIR = Path(".monkeybot") / "transcripts"
33
+
34
+
35
+ def transcript_enabled_from_env() -> bool:
36
+ """Opt-in only; default off (``MONKEYBOT_TRANSCRIPT_ENABLED``)."""
37
+ raw = os.environ.get("MONKEYBOT_TRANSCRIPT_ENABLED", "false").strip().lower()
38
+ return raw in ("1", "true", "yes", "on")
39
+
40
+
41
+ def transcript_include_live_from_env() -> bool:
42
+ """When true, write live-only streaming events too (default durable-only)."""
43
+ raw = os.environ.get("MONKEYBOT_TRANSCRIPT_INCLUDE_LIVE", "false").strip().lower()
44
+ return raw in ("1", "true", "yes", "on")
45
+
46
+
47
+ def _now_iso() -> str:
48
+ return time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime()) + f".{int(time.time() * 1000) % 1000:03d}Z"
49
+
50
+
51
+ class TranscriptWriter:
52
+ """Append-only NDJSON transcript writer for one gateway session.
53
+
54
+ One instance per ``session_id``; safe for concurrent turns on the same
55
+ session via an internal lock. All file I/O runs in ``asyncio.to_thread``.
56
+ """
57
+
58
+ def __init__(
59
+ self,
60
+ session_id: str,
61
+ *,
62
+ workspace_root: Path,
63
+ include_live: bool | None = None,
64
+ ) -> None:
65
+ self._session_id = session_id
66
+ safe_id = sanitize_path_component(session_id)
67
+ self._path = workspace_root.resolve() / _TRANSCRIPT_REL_DIR / f"{safe_id}.ndjson"
68
+ self._lock = asyncio.Lock()
69
+ self._manifest_written = False
70
+ self._seq = 0
71
+ self._include_live = (
72
+ transcript_include_live_from_env() if include_live is None else include_live
73
+ )
74
+
75
+ @property
76
+ def path(self) -> Path:
77
+ return self._path
78
+
79
+ async def ensure_manifest(self, **manifest_fields: Any) -> None:
80
+ """Write the manifest as line 1 if this is a new transcript file (idempotent)."""
81
+ if self._manifest_written:
82
+ return
83
+ async with self._lock:
84
+ if self._manifest_written:
85
+ return
86
+
87
+ def _write_if_new() -> None:
88
+ self._path.parent.mkdir(parents=True, exist_ok=True)
89
+ if self._path.exists():
90
+ return
91
+ record = {
92
+ "type": "SessionManifest",
93
+ "session_id": self._session_id,
94
+ "started_at": _now_iso(),
95
+ "durable_only": not self._include_live,
96
+ **manifest_fields,
97
+ }
98
+ line = json.dumps(record, ensure_ascii=False, separators=(",", ":"))
99
+ with self._path.open("a", encoding="utf-8") as fh:
100
+ fh.write(line + "\n")
101
+
102
+ try:
103
+ await asyncio.to_thread(_write_if_new)
104
+ except OSError:
105
+ logger.warning("transcript manifest write failed for %s", self._session_id, exc_info=True)
106
+ self._manifest_written = True
107
+
108
+ async def _append_line(self, record: dict[str, Any]) -> None:
109
+ async with self._lock:
110
+ self._seq += 1
111
+ line = json.dumps({"seq": self._seq, **record}, ensure_ascii=False, separators=(",", ":"))
112
+
113
+ def _append() -> None:
114
+ self._path.parent.mkdir(parents=True, exist_ok=True)
115
+ with self._path.open("a", encoding="utf-8") as fh:
116
+ fh.write(line + "\n")
117
+
118
+ try:
119
+ await asyncio.to_thread(_append)
120
+ except OSError:
121
+ logger.warning("transcript append failed for %s", self._session_id, exc_info=True)
122
+
123
+ async def write_user_message(self, *, request_id: str, content: str) -> None:
124
+ """Append the incoming user turn (not an ``AgentEvent``; harness-internal)."""
125
+ await self._append_line(
126
+ {"ts": _now_iso(), "type": "UserMessage", "request_id": request_id, "content": content}
127
+ )
128
+
129
+ async def write_event(self, event: AgentEvent) -> None:
130
+ """Append one harness ``AgentEvent`` (durable by default; same JSON as SSE)."""
131
+ if not self._include_live and not is_durable_event(event):
132
+ return
133
+ payload = json.loads(event_to_json(event))
134
+ await self._append_line({"ts": _now_iso(), **payload})
135
+
136
+ async def write_provider_request(
137
+ self,
138
+ *,
139
+ request_id: str,
140
+ inner_turn: int,
141
+ model: str,
142
+ messages: list[dict[str, object]],
143
+ message_offset: int = 0,
144
+ messages_reset: bool = False,
145
+ tools: list[dict[str, object]] | None = None,
146
+ thinking_budget: int | None,
147
+ ) -> None:
148
+ record: dict[str, Any] = {
149
+ "ts": _now_iso(),
150
+ "type": "ProviderRequest",
151
+ "request_id": request_id,
152
+ "inner_turn": inner_turn,
153
+ "model": model,
154
+ "message_offset": message_offset,
155
+ "messages": messages,
156
+ "thinking_budget": thinking_budget,
157
+ }
158
+ if messages_reset:
159
+ record["messages_reset"] = True
160
+ if tools is not None:
161
+ record["tools"] = tools
162
+ await self._append_line(record)
163
+
164
+ async def write_provider_response(
165
+ self,
166
+ *,
167
+ request_id: str,
168
+ inner_turn: int,
169
+ model: str,
170
+ text: str,
171
+ thinking: str,
172
+ tool_requests: list[dict[str, object]],
173
+ usage: dict[str, object],
174
+ ) -> None:
175
+ await self._append_line(
176
+ {
177
+ "ts": _now_iso(),
178
+ "type": "ProviderResponse",
179
+ "request_id": request_id,
180
+ "inner_turn": inner_turn,
181
+ "model": model,
182
+ "text": text,
183
+ "thinking": thinking,
184
+ "tool_requests": tool_requests,
185
+ "usage": usage,
186
+ }
187
+ )
188
+
189
+
190
+ __all__ = [
191
+ "TranscriptWriter",
192
+ "transcript_enabled_from_env",
193
+ "transcript_include_live_from_env",
194
+ ]
@@ -0,0 +1,149 @@
1
+ """SQLite-backed usage store."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import time
6
+
7
+ import aiosqlite
8
+
9
+ from monkeybot.core.llm.usage import Usage, UsageSummary
10
+
11
+
12
+ class SQLiteUsageStore:
13
+ """Insert ``turn_usage`` rows and compute summaries."""
14
+
15
+ def __init__(self, conn: aiosqlite.Connection) -> None:
16
+ self._conn = conn
17
+
18
+ async def record(
19
+ self,
20
+ thread_id: str,
21
+ model: str,
22
+ usage: Usage,
23
+ run_id: str | None = None,
24
+ *,
25
+ context_json: str | None = None,
26
+ ) -> None:
27
+ """Persist one usage row at ``created_at`` = now (ms)."""
28
+ now_ms = int(time.time() * 1000)
29
+ await self._conn.execute(
30
+ """
31
+ INSERT INTO turn_usage(
32
+ thread_id, run_id, model,
33
+ input_tokens, output_tokens, cached_tokens,
34
+ cost_usd, duration_ms, created_at, context_json,
35
+ estimated_prompt_tokens, cache_read_tokens, cache_creation_tokens
36
+ )
37
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
38
+ """,
39
+ (
40
+ thread_id,
41
+ run_id,
42
+ model,
43
+ usage.input_tokens,
44
+ usage.output_tokens,
45
+ usage.cached_tokens,
46
+ usage.cost_usd,
47
+ usage.duration_ms,
48
+ now_ms,
49
+ context_json,
50
+ usage.estimated_prompt_tokens,
51
+ usage.cache_read_tokens,
52
+ usage.cache_creation_tokens,
53
+ ),
54
+ )
55
+ await self._conn.commit()
56
+
57
+ async def summary(
58
+ self,
59
+ thread_id: str | None = None,
60
+ since_ms: int | None = None,
61
+ ) -> UsageSummary:
62
+ """Aggregate totals over matching rows."""
63
+ clauses: list[str] = []
64
+ params: list[object] = []
65
+ if thread_id is not None:
66
+ clauses.append("thread_id = ?")
67
+ params.append(thread_id)
68
+ if since_ms is not None:
69
+ clauses.append("created_at >= ?")
70
+ params.append(since_ms)
71
+ where_sql = ""
72
+ if clauses:
73
+ where_sql = "WHERE " + " AND ".join(clauses)
74
+
75
+ sql = f"""
76
+ SELECT
77
+ COUNT(*) AS turns,
78
+ COALESCE(SUM(input_tokens), 0) AS input_tokens,
79
+ COALESCE(SUM(output_tokens), 0) AS output_tokens,
80
+ COALESCE(SUM(cached_tokens), 0) AS cached_tokens,
81
+ COALESCE(SUM(cost_usd), 0.0) AS cost_usd,
82
+ MIN(created_at) AS period_start_ms,
83
+ MAX(created_at) AS period_end_ms,
84
+ COALESCE(SUM(cache_read_tokens), 0) AS cache_read_tokens,
85
+ COALESCE(SUM(cache_creation_tokens), 0) AS cache_creation_tokens
86
+ FROM turn_usage
87
+ {where_sql}
88
+ """
89
+
90
+ cursor = await self._conn.execute(sql, params)
91
+ row = await cursor.fetchone()
92
+ await cursor.close()
93
+ if row is None:
94
+ raise RuntimeError("usage summary query returned no row")
95
+
96
+ turns = int(row[0])
97
+ if turns == 0:
98
+ return UsageSummary(
99
+ turns=0,
100
+ input_tokens=0,
101
+ output_tokens=0,
102
+ cached_tokens=0,
103
+ cost_usd=0.0,
104
+ period_start_ms=None,
105
+ period_end_ms=None,
106
+ last_prompt_tokens=0,
107
+ last_estimated_prompt_tokens=0,
108
+ cache_read_tokens=0,
109
+ cache_creation_tokens=0,
110
+ )
111
+
112
+ period_start = row[5]
113
+ period_end = row[6]
114
+ last_pt = 0
115
+ last_est = 0
116
+ if thread_id is not None:
117
+ lp_clauses: list[str] = ["thread_id = ?"]
118
+ lp_params: list[object] = [thread_id]
119
+ if since_ms is not None:
120
+ lp_clauses.append("created_at >= ?")
121
+ lp_params.append(since_ms)
122
+ lp_where = "WHERE " + " AND ".join(lp_clauses)
123
+ cur2 = await self._conn.execute(
124
+ f"""
125
+ SELECT input_tokens, estimated_prompt_tokens FROM turn_usage
126
+ {lp_where}
127
+ ORDER BY created_at DESC
128
+ LIMIT 1
129
+ """,
130
+ lp_params,
131
+ )
132
+ row2 = await cur2.fetchone()
133
+ await cur2.close()
134
+ if row2 is not None:
135
+ last_pt = int(row2[0])
136
+ last_est = int(row2[1])
137
+ return UsageSummary(
138
+ turns=turns,
139
+ input_tokens=int(row[1]),
140
+ output_tokens=int(row[2]),
141
+ cached_tokens=int(row[3]),
142
+ cost_usd=float(row[4]),
143
+ period_start_ms=int(period_start) if period_start is not None else None,
144
+ period_end_ms=int(period_end) if period_end is not None else None,
145
+ last_prompt_tokens=last_pt,
146
+ last_estimated_prompt_tokens=last_est,
147
+ cache_read_tokens=int(row[7]),
148
+ cache_creation_tokens=int(row[8]),
149
+ )