techtide-botkit 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- botkit/__init__.py +24 -0
- botkit/agent.py +449 -0
- botkit/errors.py +6 -0
- botkit/llm.py +111 -0
- botkit/mcp.py +268 -0
- botkit/memory/__init__.py +9 -0
- botkit/memory/base.py +56 -0
- botkit/memory/memory.py +109 -0
- botkit/memory/sqlite.py +115 -0
- botkit/memory/supabase.py +188 -0
- botkit/sandbox.py +139 -0
- botkit/security.py +98 -0
- botkit/server.py +199 -0
- botkit/skill.py +137 -0
- botkit/skills/coding/SKILL.md +10 -0
- botkit/skills/data-analysis/SKILL.md +9 -0
- botkit/skills/debugging/SKILL.md +10 -0
- botkit/skills/planning/SKILL.md +10 -0
- botkit/skills/presentations/SKILL.md +10 -0
- botkit/skills/research/SKILL.md +10 -0
- botkit/skills/spreadsheets/SKILL.md +10 -0
- botkit/skills/summarizing/SKILL.md +10 -0
- botkit/skills/verify-work/SKILL.md +10 -0
- botkit/skills/writing-documents/SKILL.md +10 -0
- botkit/tool.py +198 -0
- botkit/tools/__init__.py +42 -0
- botkit/tools/docs.py +195 -0
- botkit/tools/files.py +125 -0
- botkit/tools/shell.py +23 -0
- botkit/tools/util.py +65 -0
- botkit/tools/web.py +121 -0
- techtide_botkit-0.3.0.dist-info/METADATA +196 -0
- techtide_botkit-0.3.0.dist-info/RECORD +36 -0
- techtide_botkit-0.3.0.dist-info/WHEEL +5 -0
- techtide_botkit-0.3.0.dist-info/licenses/LICENSE +21 -0
- techtide_botkit-0.3.0.dist-info/top_level.txt +1 -0
botkit/mcp.py
ADDED
|
@@ -0,0 +1,268 @@
|
|
|
1
|
+
"""MCP bridge: connect to MCP servers (stdio / HTTP / SSE) and expose their tools as botkit Tools.
|
|
2
|
+
|
|
3
|
+
Works with both the 1.x and 2.x `mcp` SDKs. The async SDK runs on one shared background event loop
|
|
4
|
+
thread, so the rest of botkit (and your code) stays plain synchronous.
|
|
5
|
+
"""
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import asyncio
|
|
9
|
+
import concurrent.futures
|
|
10
|
+
import json
|
|
11
|
+
import os
|
|
12
|
+
import re
|
|
13
|
+
import shlex
|
|
14
|
+
import threading
|
|
15
|
+
from contextlib import AsyncExitStack
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
from typing import Any
|
|
18
|
+
|
|
19
|
+
from .tool import Tool
|
|
20
|
+
|
|
21
|
+
_loop: asyncio.AbstractEventLoop | None = None
|
|
22
|
+
_loop_lock = threading.Lock()
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _get_loop() -> asyncio.AbstractEventLoop:
|
|
26
|
+
global _loop
|
|
27
|
+
with _loop_lock:
|
|
28
|
+
if _loop is None:
|
|
29
|
+
loop = asyncio.new_event_loop()
|
|
30
|
+
threading.Thread(target=loop.run_forever, name="botkit-mcp", daemon=True).start()
|
|
31
|
+
_loop = loop
|
|
32
|
+
return _loop
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _leaf(err: BaseException) -> BaseException:
|
|
36
|
+
"""Unwrap (nested) exception groups to the first real error."""
|
|
37
|
+
while getattr(err, "exceptions", None):
|
|
38
|
+
err = err.exceptions[0] # type: ignore[attr-defined]
|
|
39
|
+
return err
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _safe_name(text: str) -> str:
|
|
43
|
+
return re.sub(r"[^a-zA-Z0-9_-]", "_", text)[:64]
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
class MCP:
|
|
47
|
+
"""One MCP server.
|
|
48
|
+
|
|
49
|
+
MCP("npx", "-y", "@modelcontextprotocol/server-filesystem", ".") # local process (stdio)
|
|
50
|
+
MCP("https://example.com/mcp", headers={"Authorization": "Bearer ..."}) # remote (HTTP)
|
|
51
|
+
"""
|
|
52
|
+
|
|
53
|
+
def __init__(
|
|
54
|
+
self,
|
|
55
|
+
command: str,
|
|
56
|
+
*args: str,
|
|
57
|
+
env: dict[str, str] | None = None,
|
|
58
|
+
headers: dict[str, str] | None = None,
|
|
59
|
+
name: str | None = None,
|
|
60
|
+
timeout: float = 30.0,
|
|
61
|
+
call_timeout: float = 60.0,
|
|
62
|
+
prefix: bool = True,
|
|
63
|
+
debug: bool = False,
|
|
64
|
+
) -> None:
|
|
65
|
+
self.url = command if command.startswith(("http://", "https://")) else None
|
|
66
|
+
self.command = None if self.url else command
|
|
67
|
+
self.args = list(args)
|
|
68
|
+
self.env = env
|
|
69
|
+
self.headers = headers
|
|
70
|
+
self.name = name
|
|
71
|
+
self.timeout = timeout
|
|
72
|
+
self.call_timeout = call_timeout
|
|
73
|
+
self.prefix = prefix
|
|
74
|
+
self.debug = debug
|
|
75
|
+
self._session: Any = None
|
|
76
|
+
self._raw_tools: list[Any] = []
|
|
77
|
+
self._runner: concurrent.futures.Future | None = None
|
|
78
|
+
self._ready: concurrent.futures.Future | None = None
|
|
79
|
+
self._stop: asyncio.Event | None = None
|
|
80
|
+
|
|
81
|
+
@classmethod
|
|
82
|
+
def from_config(cls, name: str, cfg: dict) -> "MCP":
|
|
83
|
+
"""Build from one entry of the standard `mcpServers` config (same format Claude Desktop uses)."""
|
|
84
|
+
target = cfg.get("url") or cfg.get("command")
|
|
85
|
+
if not target:
|
|
86
|
+
raise ValueError(f"MCP server '{name}' needs a 'command' or a 'url'")
|
|
87
|
+
return cls(target, *cfg.get("args", []), env=cfg.get("env"), headers=cfg.get("headers"), name=name)
|
|
88
|
+
|
|
89
|
+
# ---- lifecycle ------------------------------------------------------------
|
|
90
|
+
def start(self) -> None:
|
|
91
|
+
if self._session is not None:
|
|
92
|
+
return
|
|
93
|
+
try:
|
|
94
|
+
import mcp # noqa: F401
|
|
95
|
+
except ImportError:
|
|
96
|
+
raise ImportError("MCP support is not installed. Run: pip install 'botkit[mcp]'") from None
|
|
97
|
+
self._ready = concurrent.futures.Future()
|
|
98
|
+
self._runner = asyncio.run_coroutine_threadsafe(self._main(), _get_loop())
|
|
99
|
+
try:
|
|
100
|
+
self._ready.result(timeout=self.timeout)
|
|
101
|
+
except concurrent.futures.TimeoutError:
|
|
102
|
+
self._runner.cancel()
|
|
103
|
+
raise TimeoutError(f"MCP server '{self.name}' did not start within {self.timeout:g}s") from None
|
|
104
|
+
|
|
105
|
+
def close(self) -> None:
|
|
106
|
+
if self._runner is None:
|
|
107
|
+
return
|
|
108
|
+
if self._stop is not None:
|
|
109
|
+
_get_loop().call_soon_threadsafe(self._stop.set)
|
|
110
|
+
try:
|
|
111
|
+
self._runner.result(timeout=10)
|
|
112
|
+
except (Exception, asyncio.CancelledError):
|
|
113
|
+
pass
|
|
114
|
+
self._runner, self._session, self._stop = None, None, None
|
|
115
|
+
|
|
116
|
+
def __enter__(self) -> "MCP":
|
|
117
|
+
self.start()
|
|
118
|
+
return self
|
|
119
|
+
|
|
120
|
+
def __exit__(self, *exc: Any) -> None:
|
|
121
|
+
self.close()
|
|
122
|
+
|
|
123
|
+
# ---- connection (runs on the background loop, in ONE task so anyio scopes stay valid) ----
|
|
124
|
+
async def _main(self) -> None:
|
|
125
|
+
assert self._ready is not None
|
|
126
|
+
try:
|
|
127
|
+
async with AsyncExitStack() as stack:
|
|
128
|
+
from mcp import ClientSession
|
|
129
|
+
|
|
130
|
+
read, write = await self._connect(stack)
|
|
131
|
+
session = await stack.enter_async_context(ClientSession(read, write))
|
|
132
|
+
await session.initialize()
|
|
133
|
+
raw = await self._list_all(session)
|
|
134
|
+
self._raw_tools, self._session = raw, session
|
|
135
|
+
self._stop = asyncio.Event()
|
|
136
|
+
self._ready.set_result(None)
|
|
137
|
+
await self._stop.wait()
|
|
138
|
+
except BaseException as e: # noqa: BLE001
|
|
139
|
+
if not self._ready.done():
|
|
140
|
+
self._ready.set_exception(_leaf(e))
|
|
141
|
+
if isinstance(e, asyncio.CancelledError):
|
|
142
|
+
raise
|
|
143
|
+
finally:
|
|
144
|
+
self._session = None
|
|
145
|
+
|
|
146
|
+
async def _connect(self, stack: AsyncExitStack) -> tuple[Any, Any]:
|
|
147
|
+
if self.url is None:
|
|
148
|
+
from mcp import StdioServerParameters
|
|
149
|
+
from mcp.client.stdio import stdio_client
|
|
150
|
+
|
|
151
|
+
params = StdioServerParameters(command=self.command, args=self.args, env=self.env)
|
|
152
|
+
errlog = None if self.debug else stack.enter_context(open(os.devnull, "w"))
|
|
153
|
+
cm = stdio_client(params, errlog=errlog) if errlog else stdio_client(params)
|
|
154
|
+
streams = await stack.enter_async_context(cm)
|
|
155
|
+
elif self.url.rstrip("/").endswith("/sse"):
|
|
156
|
+
from mcp.client.sse import sse_client
|
|
157
|
+
|
|
158
|
+
streams = await stack.enter_async_context(sse_client(self.url, headers=self.headers or None))
|
|
159
|
+
else:
|
|
160
|
+
streams = await stack.enter_async_context(self._http_transport(stack))
|
|
161
|
+
return streams[0], streams[1]
|
|
162
|
+
|
|
163
|
+
def _http_transport(self, stack: AsyncExitStack) -> Any:
|
|
164
|
+
try: # mcp 2.x
|
|
165
|
+
from mcp.client.streamable_http import streamable_http_client as connect
|
|
166
|
+
|
|
167
|
+
kwargs: dict[str, Any] = {}
|
|
168
|
+
if self.headers:
|
|
169
|
+
try:
|
|
170
|
+
import httpx2 as httpx_mod
|
|
171
|
+
except ImportError:
|
|
172
|
+
import httpx as httpx_mod
|
|
173
|
+
client = httpx_mod.AsyncClient(headers=self.headers, timeout=self.call_timeout)
|
|
174
|
+
stack.push_async_callback(client.aclose)
|
|
175
|
+
kwargs["http_client"] = client
|
|
176
|
+
return connect(self.url, **kwargs)
|
|
177
|
+
except ImportError: # mcp 1.x
|
|
178
|
+
from mcp.client.streamable_http import streamablehttp_client
|
|
179
|
+
|
|
180
|
+
return streamablehttp_client(self.url, headers=self.headers or None)
|
|
181
|
+
|
|
182
|
+
@staticmethod
|
|
183
|
+
async def _list_all(session: Any) -> list[Any]:
|
|
184
|
+
res = await session.list_tools()
|
|
185
|
+
tools = list(res.tools)
|
|
186
|
+
cursor = getattr(res, "nextCursor", None) or getattr(res, "next_cursor", None)
|
|
187
|
+
for _ in range(20): # follow pagination, bounded
|
|
188
|
+
if not cursor:
|
|
189
|
+
break
|
|
190
|
+
try:
|
|
191
|
+
from mcp_types import PaginatedRequestParams # mcp 2.x
|
|
192
|
+
|
|
193
|
+
res = await session.list_tools(params=PaginatedRequestParams(cursor=cursor))
|
|
194
|
+
except ImportError: # mcp 1.x
|
|
195
|
+
res = await session.list_tools(cursor=cursor)
|
|
196
|
+
tools.extend(res.tools)
|
|
197
|
+
cursor = getattr(res, "nextCursor", None) or getattr(res, "next_cursor", None)
|
|
198
|
+
return tools
|
|
199
|
+
|
|
200
|
+
# ---- using the server ------------------------------------------------------
|
|
201
|
+
def tools(self) -> list[Tool]:
|
|
202
|
+
self.start()
|
|
203
|
+
result = []
|
|
204
|
+
for raw in self._raw_tools:
|
|
205
|
+
schema = getattr(raw, "inputSchema", None) or getattr(raw, "input_schema", None) or {}
|
|
206
|
+
schema = dict(schema) if schema else {"type": "object", "properties": {}}
|
|
207
|
+
full = f"{self.name}_{raw.name}" if self.prefix and self.name else raw.name
|
|
208
|
+
result.append(
|
|
209
|
+
Tool(
|
|
210
|
+
name=_safe_name(full),
|
|
211
|
+
description=(getattr(raw, "description", None) or raw.name)[:1000],
|
|
212
|
+
parameters=schema,
|
|
213
|
+
fn=self._caller(raw.name),
|
|
214
|
+
)
|
|
215
|
+
)
|
|
216
|
+
return result
|
|
217
|
+
|
|
218
|
+
def _caller(self, tool_name: str) -> Any:
|
|
219
|
+
return lambda **kwargs: self.call(tool_name, kwargs)
|
|
220
|
+
|
|
221
|
+
def call(self, tool_name: str, arguments: dict) -> str:
|
|
222
|
+
if self._session is None: # server died or never started: try once to (re)connect
|
|
223
|
+
self.start()
|
|
224
|
+
future = asyncio.run_coroutine_threadsafe(self._session.call_tool(tool_name, arguments), _get_loop())
|
|
225
|
+
try:
|
|
226
|
+
res = future.result(timeout=self.call_timeout)
|
|
227
|
+
except concurrent.futures.TimeoutError:
|
|
228
|
+
future.cancel()
|
|
229
|
+
raise TimeoutError(f"MCP tool '{tool_name}' timed out after {self.call_timeout:g}s") from None
|
|
230
|
+
parts = []
|
|
231
|
+
for block in getattr(res, "content", None) or []:
|
|
232
|
+
text = getattr(block, "text", None)
|
|
233
|
+
parts.append(text if text is not None else f"[{getattr(block, 'type', 'content')}]")
|
|
234
|
+
text = "\n".join(parts)
|
|
235
|
+
if not text:
|
|
236
|
+
structured = getattr(res, "structuredContent", None) or getattr(res, "structured_content", None)
|
|
237
|
+
text = json.dumps(structured, default=str) if structured else "OK"
|
|
238
|
+
if getattr(res, "isError", False) or getattr(res, "is_error", False):
|
|
239
|
+
raise RuntimeError(text)
|
|
240
|
+
return text
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def normalize(spec: Any) -> list[MCP]:
|
|
244
|
+
"""Accept the many ways a user may describe MCP servers and return a list of MCP objects.
|
|
245
|
+
|
|
246
|
+
- an MCP object, or a list of them
|
|
247
|
+
- a command string: "npx -y @modelcontextprotocol/server-filesystem ."
|
|
248
|
+
- a URL string
|
|
249
|
+
- a path to a JSON file, or a dict, in the standard {"mcpServers": {name: {command, args, env}}} format
|
|
250
|
+
"""
|
|
251
|
+
if spec is None or spec is False:
|
|
252
|
+
return []
|
|
253
|
+
if isinstance(spec, MCP):
|
|
254
|
+
return [spec]
|
|
255
|
+
if isinstance(spec, (str, os.PathLike)):
|
|
256
|
+
text = str(spec)
|
|
257
|
+
if text.endswith(".json") and Path(text).expanduser().is_file():
|
|
258
|
+
return normalize(json.loads(Path(text).expanduser().read_text(encoding="utf-8")))
|
|
259
|
+
if text.startswith(("http://", "https://")):
|
|
260
|
+
return [MCP(text)]
|
|
261
|
+
return [MCP(*shlex.split(text))]
|
|
262
|
+
if isinstance(spec, dict):
|
|
263
|
+
servers = spec.get("mcpServers", spec)
|
|
264
|
+
return [MCP.from_config(name, cfg) for name, cfg in servers.items()]
|
|
265
|
+
out: list[MCP] = []
|
|
266
|
+
for item in spec:
|
|
267
|
+
out.extend(normalize(item))
|
|
268
|
+
return out
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
"""Memory: history + facts, stored in SQLite (default), Supabase, or any backend you write."""
|
|
2
|
+
from .base import Fact, Storage, trim_history
|
|
3
|
+
from .memory import Memory
|
|
4
|
+
from .sqlite import SQLiteStorage
|
|
5
|
+
from .supabase import SupabaseStorage, SupabaseTable, schema_sql
|
|
6
|
+
|
|
7
|
+
__all__ = [
|
|
8
|
+
"Memory", "Storage", "Fact", "SQLiteStorage", "SupabaseStorage", "SupabaseTable", "schema_sql", "trim_history",
|
|
9
|
+
]
|
botkit/memory/base.py
ADDED
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
"""The storage contract. Implement this class to keep botkit's memory in ANY database."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
from abc import ABC, abstractmethod
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
@dataclass
|
|
9
|
+
class Fact:
|
|
10
|
+
id: int
|
|
11
|
+
text: str
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def trim_history(messages: list[dict], limit: int | None) -> list[dict]:
|
|
15
|
+
"""Keep roughly the last `limit` messages, but never start mid tool-call (must begin with a user message)."""
|
|
16
|
+
if limit is None or len(messages) <= limit:
|
|
17
|
+
return list(messages)
|
|
18
|
+
kept = messages[-limit:]
|
|
19
|
+
while kept and kept[0].get("role") != "user":
|
|
20
|
+
kept = kept[1:]
|
|
21
|
+
return kept
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class Storage(ABC):
|
|
25
|
+
"""Where memory lives. botkit ships SQLiteStorage and SupabaseStorage; write your own for Postgres,
|
|
26
|
+
MongoDB, Redis, a file, anything. Seven small methods, all scoped by `session` (one session = one user).
|
|
27
|
+
|
|
28
|
+
Messages are plain dicts (OpenAI chat format); store them as JSON and return them unchanged, oldest first.
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
@abstractmethod
|
|
32
|
+
def add_messages(self, session: str, messages: list[dict]) -> None:
|
|
33
|
+
"""Append messages to a session's history, keeping their order."""
|
|
34
|
+
|
|
35
|
+
@abstractmethod
|
|
36
|
+
def get_messages(self, session: str, limit: int) -> list[dict]:
|
|
37
|
+
"""Return the last `limit` messages of a session, oldest first."""
|
|
38
|
+
|
|
39
|
+
@abstractmethod
|
|
40
|
+
def clear_messages(self, session: str) -> None:
|
|
41
|
+
"""Delete a session's history (facts are kept)."""
|
|
42
|
+
|
|
43
|
+
@abstractmethod
|
|
44
|
+
def add_fact(self, session: str, text: str) -> int:
|
|
45
|
+
"""Save a long-term fact and return its id."""
|
|
46
|
+
|
|
47
|
+
@abstractmethod
|
|
48
|
+
def list_facts(self, session: str, limit: int = 30) -> list[Fact]:
|
|
49
|
+
"""Return the newest `limit` facts of a session, oldest first."""
|
|
50
|
+
|
|
51
|
+
@abstractmethod
|
|
52
|
+
def delete_fact(self, session: str, fact_id: int) -> bool:
|
|
53
|
+
"""Delete one fact. Return False if it did not exist."""
|
|
54
|
+
|
|
55
|
+
def close(self) -> None:
|
|
56
|
+
"""Release connections (optional)."""
|
botkit/memory/memory.py
ADDED
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
"""Memory = conversation history + long-term facts, on top of any Storage backend."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import re
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
from ..tool import Tool, tool
|
|
8
|
+
from .base import Fact, Storage, trim_history
|
|
9
|
+
from .sqlite import SQLiteStorage
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class Memory:
|
|
13
|
+
"""
|
|
14
|
+
Memory("bot.db") # SQLite file
|
|
15
|
+
Memory(SupabaseStorage(url, key)) # Supabase
|
|
16
|
+
Memory(MyStorage(), session="user42") # your own backend
|
|
17
|
+
|
|
18
|
+
- history: the conversation so far (survives restarts)
|
|
19
|
+
- facts: durable things worth remembering ("user's name is Ahmed") that the model saves/looks up via tools
|
|
20
|
+
One storage can serve many sessions (one per user): `memory.for_session("user42")`.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
def __init__(self, storage: Storage | str | Path | None = None, session: str = "default", max_messages: int = 40) -> None:
|
|
24
|
+
if storage is None:
|
|
25
|
+
storage = SQLiteStorage(":memory:")
|
|
26
|
+
elif isinstance(storage, (str, Path)):
|
|
27
|
+
storage = SQLiteStorage(storage)
|
|
28
|
+
elif not isinstance(storage, Storage):
|
|
29
|
+
raise TypeError("memory storage must be a path or a botkit Storage object")
|
|
30
|
+
self.storage = storage
|
|
31
|
+
self.session = session
|
|
32
|
+
self.max_messages = max_messages
|
|
33
|
+
|
|
34
|
+
def for_session(self, session: str) -> "Memory":
|
|
35
|
+
"""Same storage, another conversation/user."""
|
|
36
|
+
return Memory(self.storage, session=session, max_messages=self.max_messages)
|
|
37
|
+
|
|
38
|
+
# ---- conversation history ------------------------------------------------
|
|
39
|
+
def load(self) -> list[dict]:
|
|
40
|
+
msgs = self.storage.get_messages(self.session, self.max_messages * 3)
|
|
41
|
+
return trim_history(msgs, self.max_messages)
|
|
42
|
+
|
|
43
|
+
def extend(self, messages: list[dict]) -> None:
|
|
44
|
+
if messages:
|
|
45
|
+
self.storage.add_messages(self.session, messages)
|
|
46
|
+
|
|
47
|
+
def clear_history(self) -> None:
|
|
48
|
+
self.storage.clear_messages(self.session)
|
|
49
|
+
|
|
50
|
+
# ---- long-term facts -----------------------------------------------------
|
|
51
|
+
def add_fact(self, text: str) -> int:
|
|
52
|
+
text = text.strip()
|
|
53
|
+
for f in self.storage.list_facts(self.session, 1000):
|
|
54
|
+
if f.text.lower() == text.lower():
|
|
55
|
+
return f.id
|
|
56
|
+
return self.storage.add_fact(self.session, text)
|
|
57
|
+
|
|
58
|
+
def list_facts(self, limit: int = 30) -> list[Fact]:
|
|
59
|
+
return self.storage.list_facts(self.session, limit)
|
|
60
|
+
|
|
61
|
+
def search_facts(self, query: str, limit: int = 5) -> list[Fact]:
|
|
62
|
+
words = set(re.findall(r"\w+", query.lower()))
|
|
63
|
+
scored = []
|
|
64
|
+
for f in self.list_facts(limit=1000):
|
|
65
|
+
score = len(words & set(re.findall(r"\w+", f.text.lower())))
|
|
66
|
+
if score:
|
|
67
|
+
scored.append((score, f))
|
|
68
|
+
scored.sort(key=lambda x: (-x[0], -x[1].id))
|
|
69
|
+
return [f for _, f in scored[:limit]]
|
|
70
|
+
|
|
71
|
+
def delete_fact(self, fact_id: int) -> bool:
|
|
72
|
+
return self.storage.delete_fact(self.session, fact_id)
|
|
73
|
+
|
|
74
|
+
def close(self) -> None:
|
|
75
|
+
self.storage.close()
|
|
76
|
+
|
|
77
|
+
# ---- tools exposed to the model -----------------------------------------
|
|
78
|
+
def tools(self) -> list[Tool]:
|
|
79
|
+
memory = self
|
|
80
|
+
|
|
81
|
+
@tool
|
|
82
|
+
def remember(fact: str) -> str:
|
|
83
|
+
"""Save a durable fact or preference about the user for future conversations.
|
|
84
|
+
|
|
85
|
+
Args:
|
|
86
|
+
fact: A short, self-contained statement, e.g. "User's name is Ahmed".
|
|
87
|
+
"""
|
|
88
|
+
return f"Saved (id {memory.add_fact(fact)})."
|
|
89
|
+
|
|
90
|
+
@tool
|
|
91
|
+
def recall(query: str) -> str:
|
|
92
|
+
"""Search saved facts about the user.
|
|
93
|
+
|
|
94
|
+
Args:
|
|
95
|
+
query: Keywords to look for.
|
|
96
|
+
"""
|
|
97
|
+
found = memory.search_facts(query)
|
|
98
|
+
return "\n".join(f"[{f.id}] {f.text}" for f in found) or "Nothing saved matches that."
|
|
99
|
+
|
|
100
|
+
@tool
|
|
101
|
+
def forget(fact_id: int) -> str:
|
|
102
|
+
"""Delete a saved fact by its id (see `recall`).
|
|
103
|
+
|
|
104
|
+
Args:
|
|
105
|
+
fact_id: The id of the fact to delete.
|
|
106
|
+
"""
|
|
107
|
+
return "Deleted." if memory.delete_fact(fact_id) else "No fact with that id."
|
|
108
|
+
|
|
109
|
+
return [remember, recall, forget]
|
botkit/memory/sqlite.py
ADDED
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
"""SQLite storage: the zero-setup default. One file, many sessions, plus room for your own tables."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import json
|
|
5
|
+
import re
|
|
6
|
+
import sqlite3
|
|
7
|
+
import threading
|
|
8
|
+
import time
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from typing import Any
|
|
11
|
+
|
|
12
|
+
from .base import Fact, Storage
|
|
13
|
+
|
|
14
|
+
_PREFIX = re.compile(r"[A-Za-z_][A-Za-z0-9_]{0,40}")
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class SQLiteStorage(Storage):
|
|
18
|
+
"""
|
|
19
|
+
SQLiteStorage("bot.db") # file, created if missing
|
|
20
|
+
SQLiteStorage(":memory:") # RAM only
|
|
21
|
+
|
|
22
|
+
Add your own tables to the same database:
|
|
23
|
+
|
|
24
|
+
db = SQLiteStorage("bot.db", extra_schema="CREATE TABLE IF NOT EXISTS orders(id INTEGER PRIMARY KEY, item TEXT)")
|
|
25
|
+
db.execute("INSERT INTO orders(item) VALUES(?)", ("tea",))
|
|
26
|
+
db.query("SELECT * FROM orders") # -> [{"id": 1, "item": "tea"}]
|
|
27
|
+
|
|
28
|
+
botkit's own tables are named `<prefix>messages` and `<prefix>facts` (default prefix "botkit_").
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
def __init__(self, path: str | Path = ":memory:", *, prefix: str = "botkit_", extra_schema: str | None = None) -> None:
|
|
32
|
+
if not _PREFIX.fullmatch(prefix):
|
|
33
|
+
raise ValueError("prefix may only contain letters, digits and underscores")
|
|
34
|
+
if str(path) != ":memory:":
|
|
35
|
+
p = Path(path).expanduser()
|
|
36
|
+
p.parent.mkdir(parents=True, exist_ok=True)
|
|
37
|
+
path = p
|
|
38
|
+
self.path = path
|
|
39
|
+
self._m, self._f = f"{prefix}messages", f"{prefix}facts"
|
|
40
|
+
self._lock = threading.RLock()
|
|
41
|
+
self.db = sqlite3.connect(str(path), check_same_thread=False)
|
|
42
|
+
self.db.row_factory = sqlite3.Row
|
|
43
|
+
self.db.executescript(
|
|
44
|
+
f"""
|
|
45
|
+
CREATE TABLE IF NOT EXISTS {self._m}(
|
|
46
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT, session TEXT NOT NULL, data TEXT NOT NULL, ts REAL NOT NULL);
|
|
47
|
+
CREATE INDEX IF NOT EXISTS idx_{self._m}_session ON {self._m}(session, id);
|
|
48
|
+
CREATE TABLE IF NOT EXISTS {self._f}(
|
|
49
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT, session TEXT NOT NULL, text TEXT NOT NULL, ts REAL NOT NULL);
|
|
50
|
+
CREATE INDEX IF NOT EXISTS idx_{self._f}_session ON {self._f}(session, id);
|
|
51
|
+
"""
|
|
52
|
+
)
|
|
53
|
+
if extra_schema:
|
|
54
|
+
self.db.executescript(extra_schema)
|
|
55
|
+
self.db.commit()
|
|
56
|
+
|
|
57
|
+
# ---- your own tables -----------------------------------------------------
|
|
58
|
+
def execute(self, sql: str, params: tuple | list = ()) -> int:
|
|
59
|
+
"""Run INSERT/UPDATE/DELETE/DDL with ? parameters. Returns the number of affected rows."""
|
|
60
|
+
with self._lock:
|
|
61
|
+
cur = self.db.execute(sql, params)
|
|
62
|
+
self.db.commit()
|
|
63
|
+
return cur.rowcount
|
|
64
|
+
|
|
65
|
+
def query(self, sql: str, params: tuple | list = ()) -> list[dict[str, Any]]:
|
|
66
|
+
"""Run a SELECT with ? parameters. Returns a list of dicts."""
|
|
67
|
+
with self._lock:
|
|
68
|
+
return [dict(r) for r in self.db.execute(sql, params).fetchall()]
|
|
69
|
+
|
|
70
|
+
# ---- Storage contract ------------------------------------------------------
|
|
71
|
+
def add_messages(self, session: str, messages: list[dict]) -> None:
|
|
72
|
+
now = time.time()
|
|
73
|
+
with self._lock:
|
|
74
|
+
self.db.executemany(
|
|
75
|
+
f"INSERT INTO {self._m}(session, data, ts) VALUES(?,?,?)",
|
|
76
|
+
[(session, json.dumps(m, ensure_ascii=False, default=str), now) for m in messages],
|
|
77
|
+
)
|
|
78
|
+
self.db.commit()
|
|
79
|
+
|
|
80
|
+
def get_messages(self, session: str, limit: int) -> list[dict]:
|
|
81
|
+
with self._lock:
|
|
82
|
+
rows = self.db.execute(
|
|
83
|
+
f"SELECT data FROM {self._m} WHERE session=? ORDER BY id DESC LIMIT ?", (session, limit)
|
|
84
|
+
).fetchall()
|
|
85
|
+
return [json.loads(r[0]) for r in reversed(rows)]
|
|
86
|
+
|
|
87
|
+
def clear_messages(self, session: str) -> None:
|
|
88
|
+
with self._lock:
|
|
89
|
+
self.db.execute(f"DELETE FROM {self._m} WHERE session=?", (session,))
|
|
90
|
+
self.db.commit()
|
|
91
|
+
|
|
92
|
+
def add_fact(self, session: str, text: str) -> int:
|
|
93
|
+
with self._lock:
|
|
94
|
+
cur = self.db.execute(
|
|
95
|
+
f"INSERT INTO {self._f}(session, text, ts) VALUES(?,?,?)", (session, text, time.time())
|
|
96
|
+
)
|
|
97
|
+
self.db.commit()
|
|
98
|
+
return int(cur.lastrowid)
|
|
99
|
+
|
|
100
|
+
def list_facts(self, session: str, limit: int = 30) -> list[Fact]:
|
|
101
|
+
with self._lock:
|
|
102
|
+
rows = self.db.execute(
|
|
103
|
+
f"SELECT id, text FROM {self._f} WHERE session=? ORDER BY id DESC LIMIT ?", (session, limit)
|
|
104
|
+
).fetchall()
|
|
105
|
+
return [Fact(r[0], r[1]) for r in reversed(rows)]
|
|
106
|
+
|
|
107
|
+
def delete_fact(self, session: str, fact_id: int) -> bool:
|
|
108
|
+
with self._lock:
|
|
109
|
+
cur = self.db.execute(f"DELETE FROM {self._f} WHERE id=? AND session=?", (fact_id, session))
|
|
110
|
+
self.db.commit()
|
|
111
|
+
return cur.rowcount > 0
|
|
112
|
+
|
|
113
|
+
def close(self) -> None:
|
|
114
|
+
with self._lock:
|
|
115
|
+
self.db.close()
|