techtide-botkit 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- botkit/__init__.py +24 -0
- botkit/agent.py +449 -0
- botkit/errors.py +6 -0
- botkit/llm.py +111 -0
- botkit/mcp.py +268 -0
- botkit/memory/__init__.py +9 -0
- botkit/memory/base.py +56 -0
- botkit/memory/memory.py +109 -0
- botkit/memory/sqlite.py +115 -0
- botkit/memory/supabase.py +188 -0
- botkit/sandbox.py +139 -0
- botkit/security.py +98 -0
- botkit/server.py +199 -0
- botkit/skill.py +137 -0
- botkit/skills/coding/SKILL.md +10 -0
- botkit/skills/data-analysis/SKILL.md +9 -0
- botkit/skills/debugging/SKILL.md +10 -0
- botkit/skills/planning/SKILL.md +10 -0
- botkit/skills/presentations/SKILL.md +10 -0
- botkit/skills/research/SKILL.md +10 -0
- botkit/skills/spreadsheets/SKILL.md +10 -0
- botkit/skills/summarizing/SKILL.md +10 -0
- botkit/skills/verify-work/SKILL.md +10 -0
- botkit/skills/writing-documents/SKILL.md +10 -0
- botkit/tool.py +198 -0
- botkit/tools/__init__.py +42 -0
- botkit/tools/docs.py +195 -0
- botkit/tools/files.py +125 -0
- botkit/tools/shell.py +23 -0
- botkit/tools/util.py +65 -0
- botkit/tools/web.py +121 -0
- techtide_botkit-0.3.0.dist-info/METADATA +196 -0
- techtide_botkit-0.3.0.dist-info/RECORD +36 -0
- techtide_botkit-0.3.0.dist-info/WHEEL +5 -0
- techtide_botkit-0.3.0.dist-info/licenses/LICENSE +21 -0
- techtide_botkit-0.3.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
"""Supabase storage over its REST API (PostgREST). No extra packages needed.
|
|
2
|
+
|
|
3
|
+
1. In Supabase: SQL Editor -> run the output of `python -m botkit.memory.supabase`.
|
|
4
|
+
2. Use the project URL and the **service_role** key (server side only, never in a browser/app).
|
|
5
|
+
|
|
6
|
+
storage = SupabaseStorage() # reads SUPABASE_URL and SUPABASE_KEY
|
|
7
|
+
storage = SupabaseStorage("https://xyz.supabase.co", "service-role-key")
|
|
8
|
+
bot = Agent(memory=storage)
|
|
9
|
+
|
|
10
|
+
Your own tables work the same way: storage.table("orders").insert({"item": "tea"})
|
|
11
|
+
"""
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import json
|
|
15
|
+
import os
|
|
16
|
+
import re
|
|
17
|
+
import urllib.error
|
|
18
|
+
import urllib.parse
|
|
19
|
+
import urllib.request
|
|
20
|
+
from typing import Any
|
|
21
|
+
|
|
22
|
+
from ..errors import BotError
|
|
23
|
+
from .base import Fact, Storage
|
|
24
|
+
|
|
25
|
+
_IDENT = re.compile(r"[A-Za-z_][A-Za-z0-9_]{0,62}")
|
|
26
|
+
_ORDER = re.compile(r"[A-Za-z_][A-Za-z0-9_]*(\.(asc|desc))?")
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def schema_sql(prefix: str = "botkit_") -> str:
|
|
30
|
+
"""SQL that creates botkit's two tables. Run it once in the Supabase SQL editor."""
|
|
31
|
+
if not _IDENT.fullmatch(prefix):
|
|
32
|
+
raise ValueError("prefix may only contain letters, digits and underscores")
|
|
33
|
+
m, f = f"{prefix}messages", f"{prefix}facts"
|
|
34
|
+
return f"""create table if not exists {m} (
|
|
35
|
+
id bigint generated always as identity primary key,
|
|
36
|
+
session text not null,
|
|
37
|
+
data jsonb not null,
|
|
38
|
+
created_at timestamptz not null default now()
|
|
39
|
+
);
|
|
40
|
+
create index if not exists {m}_session_idx on {m} (session, id);
|
|
41
|
+
create table if not exists {f} (
|
|
42
|
+
id bigint generated always as identity primary key,
|
|
43
|
+
session text not null,
|
|
44
|
+
text text not null,
|
|
45
|
+
created_at timestamptz not null default now()
|
|
46
|
+
);
|
|
47
|
+
create index if not exists {f}_session_idx on {f} (session, id);
|
|
48
|
+
-- Row level security on, no policies: only the service_role key can read/write these tables.
|
|
49
|
+
alter table {m} enable row level security;
|
|
50
|
+
alter table {f} enable row level security;
|
|
51
|
+
"""
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _filter_params(filters: dict[str, Any] | None) -> list[tuple[str, str]]:
|
|
55
|
+
"""{"user": "ahmed", "age": ("gt", 18)} -> [("user", "eq.ahmed"), ("age", "gt.18")]"""
|
|
56
|
+
out: list[tuple[str, str]] = []
|
|
57
|
+
for col, cond in (filters or {}).items():
|
|
58
|
+
if not _IDENT.fullmatch(str(col)):
|
|
59
|
+
raise ValueError(f"invalid column name {col!r}")
|
|
60
|
+
if isinstance(cond, tuple):
|
|
61
|
+
op, value = cond
|
|
62
|
+
if op not in ("eq", "neq", "gt", "gte", "lt", "lte", "like", "ilike"):
|
|
63
|
+
raise ValueError(f"unsupported operator {op!r}")
|
|
64
|
+
else:
|
|
65
|
+
op, value = "eq", cond
|
|
66
|
+
out.append((col, "is.null" if value is None else f"{op}.{value}"))
|
|
67
|
+
return out
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
class SupabaseTable:
|
|
71
|
+
"""Tiny helper for your own tables: select / insert / update / delete."""
|
|
72
|
+
|
|
73
|
+
def __init__(self, storage: "SupabaseStorage", name: str) -> None:
|
|
74
|
+
if not _IDENT.fullmatch(name):
|
|
75
|
+
raise ValueError(f"invalid table name {name!r}")
|
|
76
|
+
self._s, self.name = storage, name
|
|
77
|
+
|
|
78
|
+
def select(self, filters: dict | None = None, columns: str = "*", order: str | None = None,
|
|
79
|
+
limit: int | None = None) -> list[dict]:
|
|
80
|
+
params = [("select", columns), *_filter_params(filters)]
|
|
81
|
+
if order:
|
|
82
|
+
if not _ORDER.fullmatch(order):
|
|
83
|
+
raise ValueError("order must look like 'id' or 'id.desc'")
|
|
84
|
+
params.append(("order", order))
|
|
85
|
+
if limit is not None:
|
|
86
|
+
params.append(("limit", str(int(limit))))
|
|
87
|
+
return self._s._request("GET", self.name, params) or []
|
|
88
|
+
|
|
89
|
+
def insert(self, rows: dict | list[dict], returning: bool = True) -> list[dict]:
|
|
90
|
+
return self._s._request("POST", self.name, None, rows, returning) or []
|
|
91
|
+
|
|
92
|
+
def update(self, values: dict, filters: dict) -> list[dict]:
|
|
93
|
+
if not filters:
|
|
94
|
+
raise ValueError("update() needs filters (refusing to change every row)")
|
|
95
|
+
return self._s._request("PATCH", self.name, _filter_params(filters), values, True) or []
|
|
96
|
+
|
|
97
|
+
def delete(self, filters: dict) -> list[dict]:
|
|
98
|
+
if not filters:
|
|
99
|
+
raise ValueError("delete() needs filters (refusing to delete every row)")
|
|
100
|
+
return self._s._request("DELETE", self.name, _filter_params(filters), None, True) or []
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
class SupabaseStorage(Storage):
|
|
104
|
+
def __init__(
|
|
105
|
+
self,
|
|
106
|
+
url: str | None = None,
|
|
107
|
+
key: str | None = None,
|
|
108
|
+
*,
|
|
109
|
+
prefix: str = "botkit_",
|
|
110
|
+
timeout: float = 15.0,
|
|
111
|
+
) -> None:
|
|
112
|
+
url = url or os.environ.get("SUPABASE_URL")
|
|
113
|
+
key = key or os.environ.get("SUPABASE_KEY")
|
|
114
|
+
if not url or not key:
|
|
115
|
+
raise BotError("Supabase needs a URL and a key: pass them or set SUPABASE_URL and SUPABASE_KEY.")
|
|
116
|
+
if not url.startswith(("http://", "https://")):
|
|
117
|
+
raise BotError("Supabase URL must start with https://")
|
|
118
|
+
url = url.rstrip("/")
|
|
119
|
+
self.base = url if url.endswith("/rest/v1") else url + "/rest/v1"
|
|
120
|
+
self.key, self.timeout = key, timeout
|
|
121
|
+
if not _IDENT.fullmatch(prefix):
|
|
122
|
+
raise ValueError("prefix may only contain letters, digits and underscores")
|
|
123
|
+
self._m, self._f = f"{prefix}messages", f"{prefix}facts"
|
|
124
|
+
self.prefix = prefix
|
|
125
|
+
|
|
126
|
+
@staticmethod
|
|
127
|
+
def schema_sql(prefix: str = "botkit_") -> str:
|
|
128
|
+
return schema_sql(prefix)
|
|
129
|
+
|
|
130
|
+
def table(self, name: str) -> SupabaseTable:
|
|
131
|
+
return SupabaseTable(self, name)
|
|
132
|
+
|
|
133
|
+
# ---- HTTP ------------------------------------------------------------------
|
|
134
|
+
def _request(self, method: str, table: str, params: list[tuple[str, str]] | None = None,
|
|
135
|
+
body: Any = None, returning: bool = False) -> Any:
|
|
136
|
+
query = urllib.parse.urlencode(params or [], quote_via=urllib.parse.quote)
|
|
137
|
+
url = f"{self.base}/{table}" + (f"?{query}" if query else "")
|
|
138
|
+
headers = {
|
|
139
|
+
"apikey": self.key,
|
|
140
|
+
"Authorization": f"Bearer {self.key}",
|
|
141
|
+
"Content-Type": "application/json",
|
|
142
|
+
"Accept": "application/json",
|
|
143
|
+
}
|
|
144
|
+
if returning:
|
|
145
|
+
headers["Prefer"] = "return=representation"
|
|
146
|
+
data = json.dumps(body, ensure_ascii=False, default=str).encode("utf-8") if body is not None else None
|
|
147
|
+
req = urllib.request.Request(url, data=data, headers=headers, method=method)
|
|
148
|
+
try:
|
|
149
|
+
with urllib.request.urlopen(req, timeout=self.timeout) as resp:
|
|
150
|
+
raw = resp.read()
|
|
151
|
+
except urllib.error.HTTPError as e:
|
|
152
|
+
text = e.read().decode("utf-8", "replace")[:400]
|
|
153
|
+
if e.code == 404 and ("PGRST205" in text or "schema cache" in text):
|
|
154
|
+
raise BotError(
|
|
155
|
+
f"Supabase table '{table}' not found. Create botkit's tables first: run "
|
|
156
|
+
"SupabaseStorage.schema_sql() in the Supabase SQL editor."
|
|
157
|
+
) from None
|
|
158
|
+
raise BotError(f"Supabase error {e.code}: {text}") from None
|
|
159
|
+
except urllib.error.URLError as e:
|
|
160
|
+
raise BotError(f"Could not reach Supabase: {e.reason}") from None
|
|
161
|
+
return json.loads(raw) if raw.strip() else None
|
|
162
|
+
|
|
163
|
+
# ---- Storage contract --------------------------------------------------------
|
|
164
|
+
def add_messages(self, session: str, messages: list[dict]) -> None:
|
|
165
|
+
if messages:
|
|
166
|
+
self.table(self._m).insert([{"session": session, "data": m} for m in messages], returning=False)
|
|
167
|
+
|
|
168
|
+
def get_messages(self, session: str, limit: int) -> list[dict]:
|
|
169
|
+
rows = self.table(self._m).select({"session": session}, "data", order="id.desc", limit=limit)
|
|
170
|
+
return [r["data"] for r in reversed(rows)]
|
|
171
|
+
|
|
172
|
+
def clear_messages(self, session: str) -> None:
|
|
173
|
+
self.table(self._m).delete({"session": session})
|
|
174
|
+
|
|
175
|
+
def add_fact(self, session: str, text: str) -> int:
|
|
176
|
+
rows = self.table(self._f).insert({"session": session, "text": text})
|
|
177
|
+
return int(rows[0]["id"])
|
|
178
|
+
|
|
179
|
+
def list_facts(self, session: str, limit: int = 30) -> list[Fact]:
|
|
180
|
+
rows = self.table(self._f).select({"session": session}, "id,text", order="id.desc", limit=limit)
|
|
181
|
+
return [Fact(int(r["id"]), r["text"]) for r in reversed(rows)]
|
|
182
|
+
|
|
183
|
+
def delete_fact(self, session: str, fact_id: int) -> bool:
|
|
184
|
+
return bool(self.table(self._f).delete({"id": int(fact_id), "session": session}))
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
if __name__ == "__main__": # python -m botkit.memory.supabase
|
|
188
|
+
print(schema_sql())
|
botkit/sandbox.py
ADDED
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
"""Where shell commands run. You choose: Agent(shell="local" | "docker" | Sandbox(...)). Default: no shell."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import copy
|
|
5
|
+
import os
|
|
6
|
+
import signal
|
|
7
|
+
import subprocess
|
|
8
|
+
import uuid
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
from .errors import BotError
|
|
12
|
+
from .security import Policy
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _clip(data: bytes, limit: int) -> str:
|
|
16
|
+
text = data.decode("utf-8", errors="replace")
|
|
17
|
+
if len(text) > limit:
|
|
18
|
+
text = text[:limit] + f"\n...[truncated {len(text) - limit} characters]"
|
|
19
|
+
return text
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class Sandbox:
|
|
23
|
+
policy: Policy
|
|
24
|
+
|
|
25
|
+
def run(self, command: str, timeout: float) -> tuple[int, str]: # pragma: no cover - interface
|
|
26
|
+
raise NotImplementedError
|
|
27
|
+
|
|
28
|
+
def rebind(self, policy: Policy) -> "Sandbox":
|
|
29
|
+
clone = copy.copy(self)
|
|
30
|
+
clone.policy = policy
|
|
31
|
+
return clone
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class LocalSandbox(Sandbox):
|
|
35
|
+
"""Runs commands on this machine, in the workspace folder, with a scrubbed environment.
|
|
36
|
+
|
|
37
|
+
NOT isolated: a determined user can still do damage the filter does not foresee.
|
|
38
|
+
Use it for your own machine / trusted users; use DockerSandbox for anyone else.
|
|
39
|
+
"""
|
|
40
|
+
|
|
41
|
+
def __init__(self, policy: Policy | None = None, env: dict[str, str] | None = None) -> None:
|
|
42
|
+
self.policy = policy # type: ignore[assignment]
|
|
43
|
+
self.env = env
|
|
44
|
+
|
|
45
|
+
def run(self, command: str, timeout: float) -> tuple[int, str]:
|
|
46
|
+
self.policy.check_command(command, confine=self.policy.confine_shell)
|
|
47
|
+
root = self.policy.workdir.resolve()
|
|
48
|
+
root.mkdir(parents=True, exist_ok=True)
|
|
49
|
+
proc = subprocess.Popen(
|
|
50
|
+
["/bin/sh", "-c", command],
|
|
51
|
+
cwd=root,
|
|
52
|
+
env=self.policy.safe_env(self.env),
|
|
53
|
+
stdin=subprocess.DEVNULL,
|
|
54
|
+
stdout=subprocess.PIPE,
|
|
55
|
+
stderr=subprocess.STDOUT,
|
|
56
|
+
start_new_session=True,
|
|
57
|
+
)
|
|
58
|
+
try:
|
|
59
|
+
out, _ = proc.communicate(timeout=timeout)
|
|
60
|
+
except subprocess.TimeoutExpired:
|
|
61
|
+
os.killpg(proc.pid, signal.SIGKILL)
|
|
62
|
+
out, _ = proc.communicate()
|
|
63
|
+
return 124, _clip(out, self.policy.max_output) + f"\n[killed: timed out after {timeout:g}s]"
|
|
64
|
+
return proc.returncode, _clip(out, self.policy.max_output)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
class DockerSandbox(Sandbox):
|
|
68
|
+
"""Runs each command in a throwaway container: only the workspace is mounted, no network by default,
|
|
69
|
+
read-only root filesystem, no capabilities, memory/cpu/process limits."""
|
|
70
|
+
|
|
71
|
+
def __init__(
|
|
72
|
+
self,
|
|
73
|
+
policy: Policy | None = None,
|
|
74
|
+
image: str = "python:3.12-slim",
|
|
75
|
+
network: bool = False,
|
|
76
|
+
memory: str = "512m",
|
|
77
|
+
cpus: str = "1",
|
|
78
|
+
env: dict[str, str] | None = None,
|
|
79
|
+
) -> None:
|
|
80
|
+
self.policy = policy # type: ignore[assignment]
|
|
81
|
+
self.image, self.network, self.memory, self.cpus, self.env = image, network, memory, cpus, env or {}
|
|
82
|
+
|
|
83
|
+
def check(self) -> None:
|
|
84
|
+
try:
|
|
85
|
+
r = subprocess.run(
|
|
86
|
+
["docker", "version", "--format", "{{.Server.Version}}"], capture_output=True, text=True, timeout=10
|
|
87
|
+
)
|
|
88
|
+
except (FileNotFoundError, subprocess.TimeoutExpired) as e:
|
|
89
|
+
raise BotError(f"Docker is not available: {e}") from e
|
|
90
|
+
if r.returncode != 0:
|
|
91
|
+
raise BotError("Docker is not available: " + (r.stderr.strip() or "is the daemon running?"))
|
|
92
|
+
|
|
93
|
+
def build_argv(self, command: str, name: str) -> list[str]:
|
|
94
|
+
argv = [
|
|
95
|
+
"docker", "run", "--rm", "--name", name,
|
|
96
|
+
"--network", "bridge" if self.network else "none",
|
|
97
|
+
"--memory", self.memory, "--cpus", self.cpus, "--pids-limit", "256",
|
|
98
|
+
"--cap-drop", "ALL", "--security-opt", "no-new-privileges",
|
|
99
|
+
"--read-only", "--tmpfs", "/tmp",
|
|
100
|
+
"-v", f"{self.policy.workdir.resolve()}:/work", "-w", "/work",
|
|
101
|
+
]
|
|
102
|
+
if hasattr(os, "getuid"):
|
|
103
|
+
argv += ["--user", f"{os.getuid()}:{os.getgid()}"]
|
|
104
|
+
for key, value in self.env.items():
|
|
105
|
+
argv += ["-e", f"{key}={value}"]
|
|
106
|
+
return argv + [self.image, "sh", "-c", command]
|
|
107
|
+
|
|
108
|
+
def run(self, command: str, timeout: float) -> tuple[int, str]:
|
|
109
|
+
self.policy.check_command(command, confine=False)
|
|
110
|
+
self.policy.workdir.resolve().mkdir(parents=True, exist_ok=True)
|
|
111
|
+
name = f"botkit-{uuid.uuid4().hex[:12]}"
|
|
112
|
+
proc = subprocess.Popen(
|
|
113
|
+
self.build_argv(command, name), stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.STDOUT
|
|
114
|
+
)
|
|
115
|
+
try:
|
|
116
|
+
out, _ = proc.communicate(timeout=timeout)
|
|
117
|
+
except subprocess.TimeoutExpired:
|
|
118
|
+
subprocess.run(["docker", "kill", name], capture_output=True, timeout=15)
|
|
119
|
+
proc.kill()
|
|
120
|
+
out, _ = proc.communicate()
|
|
121
|
+
return 124, _clip(out, self.policy.max_output) + f"\n[killed: timed out after {timeout:g}s]"
|
|
122
|
+
return proc.returncode, _clip(out, self.policy.max_output)
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def make_sandbox(spec: Any, policy: Policy) -> Sandbox | None:
|
|
126
|
+
"""shell=None/False -> no shell | "local" | "docker" | {"env": "docker", "image": "node:20", "network": True} | Sandbox."""
|
|
127
|
+
if spec in (None, False):
|
|
128
|
+
return None
|
|
129
|
+
if isinstance(spec, Sandbox):
|
|
130
|
+
return spec.rebind(policy)
|
|
131
|
+
options: dict[str, Any] = {}
|
|
132
|
+
if isinstance(spec, dict):
|
|
133
|
+
options = dict(spec)
|
|
134
|
+
spec = options.pop("env", None)
|
|
135
|
+
if spec == "local":
|
|
136
|
+
return LocalSandbox(policy, **options)
|
|
137
|
+
if spec == "docker":
|
|
138
|
+
return DockerSandbox(policy, **options)
|
|
139
|
+
raise ValueError(f"Unknown shell environment {spec!r}. Use None, 'local', 'docker', a dict or a Sandbox.")
|
botkit/security.py
ADDED
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
"""Security policy, enforced in code (not by asking the model nicely).
|
|
2
|
+
|
|
3
|
+
What it guarantees:
|
|
4
|
+
- File tools can only touch files inside the workspace folder (symlinks and `..` are resolved first).
|
|
5
|
+
- Shell commands run with a scrubbed environment (your API keys are NOT passed to them).
|
|
6
|
+
- Obviously destructive commands are blocked, and in local mode so are paths outside the workspace.
|
|
7
|
+
- Secret values (API keys, tokens) are redacted from anything a tool returns to the model.
|
|
8
|
+
|
|
9
|
+
What it does NOT guarantee: the command filter is a speed bump, not a jail. For untrusted users
|
|
10
|
+
use the Docker sandbox, which gives real isolation (no network, read-only root, memory/cpu limits).
|
|
11
|
+
"""
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import os
|
|
15
|
+
import re
|
|
16
|
+
from dataclasses import dataclass, field
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
|
|
19
|
+
# (regex, reason) - blocked in every environment
|
|
20
|
+
DEFAULT_DENY: list[tuple[str, str]] = [
|
|
21
|
+
(r"\b(sudo|doas)\b|(^|[;&|]\s*)su(\s|$)", "privilege escalation"),
|
|
22
|
+
(r"\brm\b[^;&|\n]*\s(/|/\*|~|~/|\$HOME)(\s|$)", "deleting outside the workspace"),
|
|
23
|
+
(r":\s*\(\s*\)\s*\{", "fork bomb"),
|
|
24
|
+
(r"\bmkfs(\.\w+)?\b", "formatting a disk"),
|
|
25
|
+
(r"\bdd\b[^\n]*\bof=/dev/", "writing to a device"),
|
|
26
|
+
(r">\s*/dev/(sd|nvme|hd|vd)", "writing to a device"),
|
|
27
|
+
(r"\b(shutdown|reboot|halt|poweroff)\b", "power control"),
|
|
28
|
+
(r"\b(curl|wget)\b[^|\n]*\|\s*(sudo\s+)?(ba|z)?sh\b", "piping a download into a shell"),
|
|
29
|
+
(r"/dev/tcp/|\b(nc|ncat|netcat)\b[^\n]*\s-e\b", "reverse shell"),
|
|
30
|
+
]
|
|
31
|
+
|
|
32
|
+
_ABS_PATH = re.compile(r"(?<![\w.:/-])(/[\w.-][\w./-]*)")
|
|
33
|
+
_DOTDOT = re.compile(r"(^|[\s/'\"=])\.\.($|[\s/'\"])")
|
|
34
|
+
_ALLOWED_ABS = ("/usr/", "/bin/", "/lib", "/dev/null", "/dev/stdout", "/dev/stderr")
|
|
35
|
+
_SECRET_NAME = re.compile(r"KEY|TOKEN|SECRET|PASSWORD|PASSWD|CREDENTIAL", re.I)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@dataclass
|
|
39
|
+
class Policy:
|
|
40
|
+
"""Limits applied to every built-in tool.
|
|
41
|
+
|
|
42
|
+
Args:
|
|
43
|
+
workdir: the only folder the file tools and shell may use.
|
|
44
|
+
deny: extra regexes; a shell command matching any is refused (e.g. ["git push", "pip install"]).
|
|
45
|
+
allow_network: set False to remove the web tool.
|
|
46
|
+
confine_shell: in local mode, refuse commands that mention paths outside the workspace.
|
|
47
|
+
"""
|
|
48
|
+
|
|
49
|
+
workdir: Path
|
|
50
|
+
deny: list[str] = field(default_factory=list)
|
|
51
|
+
secrets: list[str] = field(default_factory=list)
|
|
52
|
+
allow_network: bool = True
|
|
53
|
+
confine_shell: bool = True
|
|
54
|
+
max_file_bytes: int = 5_000_000
|
|
55
|
+
max_output: int = 20_000
|
|
56
|
+
shell_timeout_max: int = 300
|
|
57
|
+
|
|
58
|
+
def resolve(self, path: str) -> Path:
|
|
59
|
+
"""Map a user/model supplied path to a real path inside the workspace, or raise PermissionError."""
|
|
60
|
+
root = self.workdir.resolve()
|
|
61
|
+
root.mkdir(parents=True, exist_ok=True)
|
|
62
|
+
target = (root / str(path)).resolve()
|
|
63
|
+
if target != root and root not in target.parents:
|
|
64
|
+
raise PermissionError(f"path '{path}' is outside the workspace")
|
|
65
|
+
return target
|
|
66
|
+
|
|
67
|
+
def check_command(self, command: str, confine: bool = True) -> None:
|
|
68
|
+
for pattern, reason in DEFAULT_DENY:
|
|
69
|
+
if re.search(pattern, command, re.I):
|
|
70
|
+
raise PermissionError(f"command blocked ({reason})")
|
|
71
|
+
for pattern in self.deny:
|
|
72
|
+
if re.search(pattern, command, re.I):
|
|
73
|
+
raise PermissionError(f"command blocked by app rule: {pattern}")
|
|
74
|
+
if not confine:
|
|
75
|
+
return
|
|
76
|
+
if "~" in command or "$HOME" in command or "${HOME}" in command or _DOTDOT.search(command):
|
|
77
|
+
raise PermissionError("command blocked (paths outside the workspace are not allowed)")
|
|
78
|
+
root = str(self.workdir.resolve())
|
|
79
|
+
for m in _ABS_PATH.finditer(command):
|
|
80
|
+
p = m.group(1)
|
|
81
|
+
if not (p == root or p.startswith(root + "/") or p.startswith(_ALLOWED_ABS)):
|
|
82
|
+
raise PermissionError(f"command blocked (path '{p}' is outside the workspace)")
|
|
83
|
+
|
|
84
|
+
def safe_env(self, extra: dict[str, str] | None = None) -> dict[str, str]:
|
|
85
|
+
env = {
|
|
86
|
+
"PATH": os.environ.get("PATH", "/usr/local/bin:/usr/bin:/bin"),
|
|
87
|
+
"HOME": str(self.workdir.resolve()),
|
|
88
|
+
"LANG": "C.UTF-8",
|
|
89
|
+
}
|
|
90
|
+
env.update(extra or {})
|
|
91
|
+
return env
|
|
92
|
+
|
|
93
|
+
def redact(self, text: str) -> str:
|
|
94
|
+
values = {s for s in self.secrets if s and len(s) >= 8}
|
|
95
|
+
values |= {v for k, v in os.environ.items() if _SECRET_NAME.search(k) and len(v) >= 8}
|
|
96
|
+
for v in sorted(values, key=len, reverse=True):
|
|
97
|
+
text = text.replace(v, "[REDACTED]")
|
|
98
|
+
return text
|
botkit/server.py
ADDED
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
"""A small HTTP API so any web/mobile UI can talk to your bot (standard library only).
|
|
2
|
+
|
|
3
|
+
POST /chat {"message": "...", "session": "user42"} -> {"reply": "...", "files": [{"name", "url"}]}
|
|
4
|
+
GET /files/<session>/<path> -> download a file the bot created
|
|
5
|
+
GET /health
|
|
6
|
+
|
|
7
|
+
Each `session` is an independent user: own history, own workspace folder. Set `token=` and send
|
|
8
|
+
`Authorization: Bearer <token>` (file downloads also accept `?token=`). Listening on anything other than
|
|
9
|
+
localhost without a token is refused.
|
|
10
|
+
"""
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import dataclasses
|
|
14
|
+
import hmac
|
|
15
|
+
import json
|
|
16
|
+
import mimetypes
|
|
17
|
+
import os
|
|
18
|
+
import re
|
|
19
|
+
import sys
|
|
20
|
+
import threading
|
|
21
|
+
from collections import OrderedDict
|
|
22
|
+
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
23
|
+
from typing import Any
|
|
24
|
+
from urllib.parse import parse_qs, quote, unquote, urlparse
|
|
25
|
+
|
|
26
|
+
from .errors import BotError, LLMError
|
|
27
|
+
|
|
28
|
+
_SESSION = re.compile(r"[A-Za-z0-9_-]{1,64}")
|
|
29
|
+
_MAX_BODY = 1_000_000
|
|
30
|
+
_MAX_MESSAGE = 20_000
|
|
31
|
+
_MAX_DOWNLOAD = 50_000_000
|
|
32
|
+
_LOCAL = ("127.0.0.1", "localhost", "::1")
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _snapshot(root: str) -> dict[str, tuple[int, int]]:
|
|
36
|
+
out: dict[str, tuple[int, int]] = {}
|
|
37
|
+
for base, _dirs, files in os.walk(root):
|
|
38
|
+
for name in files:
|
|
39
|
+
full = os.path.join(base, name)
|
|
40
|
+
try:
|
|
41
|
+
st = os.stat(full)
|
|
42
|
+
except OSError:
|
|
43
|
+
continue
|
|
44
|
+
out[os.path.relpath(full, root).replace(os.sep, "/")] = (st.st_mtime_ns, st.st_size)
|
|
45
|
+
if len(out) >= 5000:
|
|
46
|
+
return out
|
|
47
|
+
return out
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class _Sessions:
|
|
51
|
+
def __init__(self, agent: Any, cap: int) -> None:
|
|
52
|
+
self.base, self.cap = agent, cap
|
|
53
|
+
self._agents: OrderedDict[str, Any] = OrderedDict()
|
|
54
|
+
self._lock = threading.Lock()
|
|
55
|
+
|
|
56
|
+
def get(self, sid: str) -> Any:
|
|
57
|
+
with self._lock:
|
|
58
|
+
agent = self._agents.get(sid)
|
|
59
|
+
if agent is None:
|
|
60
|
+
agent = self._agents[sid] = self.base.fork(sid)
|
|
61
|
+
while len(self._agents) > self.cap:
|
|
62
|
+
self._agents.popitem(last=False)
|
|
63
|
+
else:
|
|
64
|
+
self._agents.move_to_end(sid)
|
|
65
|
+
return agent
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _make_handler(sessions: _Sessions, token: str | None, cors_origins: list[str] | None) -> type:
|
|
69
|
+
class Handler(BaseHTTPRequestHandler):
|
|
70
|
+
server_version = "botkit"
|
|
71
|
+
|
|
72
|
+
def log_message(self, *args: Any) -> None: # keep the console quiet
|
|
73
|
+
pass
|
|
74
|
+
|
|
75
|
+
# -- helpers
|
|
76
|
+
def _cors(self) -> None:
|
|
77
|
+
origin = self.headers.get("Origin")
|
|
78
|
+
if cors_origins and origin and ("*" in cors_origins or origin in cors_origins):
|
|
79
|
+
self.send_header("Access-Control-Allow-Origin", "*" if "*" in cors_origins else origin)
|
|
80
|
+
self.send_header("Vary", "Origin")
|
|
81
|
+
self.send_header("Access-Control-Allow-Headers", "Authorization, Content-Type")
|
|
82
|
+
self.send_header("Access-Control-Allow-Methods", "GET, POST, OPTIONS")
|
|
83
|
+
|
|
84
|
+
def _json(self, code: int, obj: Any) -> None:
|
|
85
|
+
body = json.dumps(obj, ensure_ascii=False).encode("utf-8")
|
|
86
|
+
self.send_response(code)
|
|
87
|
+
self.send_header("Content-Type", "application/json; charset=utf-8")
|
|
88
|
+
self.send_header("Content-Length", str(len(body)))
|
|
89
|
+
self.send_header("X-Content-Type-Options", "nosniff")
|
|
90
|
+
self._cors()
|
|
91
|
+
self.end_headers()
|
|
92
|
+
self.wfile.write(body)
|
|
93
|
+
|
|
94
|
+
def _authed(self, query: dict[str, list[str]]) -> bool:
|
|
95
|
+
if token is None:
|
|
96
|
+
return True
|
|
97
|
+
supplied = self.headers.get("Authorization", "")
|
|
98
|
+
supplied = supplied[7:] if supplied.lower().startswith("bearer ") else (query.get("token") or [""])[0]
|
|
99
|
+
return hmac.compare_digest(supplied.encode(), token.encode())
|
|
100
|
+
|
|
101
|
+
# -- routes
|
|
102
|
+
def do_OPTIONS(self) -> None:
|
|
103
|
+
self.send_response(204)
|
|
104
|
+
self._cors()
|
|
105
|
+
self.end_headers()
|
|
106
|
+
|
|
107
|
+
def do_GET(self) -> None:
|
|
108
|
+
url = urlparse(self.path)
|
|
109
|
+
query = parse_qs(url.query)
|
|
110
|
+
if url.path == "/health":
|
|
111
|
+
return self._json(200, {"ok": True})
|
|
112
|
+
if not self._authed(query):
|
|
113
|
+
return self._json(401, {"error": "unauthorized"})
|
|
114
|
+
m = re.fullmatch(r"/files/([^/]+)/(.+)", url.path)
|
|
115
|
+
if not m or not _SESSION.fullmatch(m.group(1)):
|
|
116
|
+
return self._json(404, {"error": "not found"})
|
|
117
|
+
policy = dataclasses.replace(sessions.base.policy, workdir=sessions.base.policy.workdir / m.group(1))
|
|
118
|
+
try:
|
|
119
|
+
target = policy.resolve(unquote(m.group(2)))
|
|
120
|
+
except PermissionError:
|
|
121
|
+
return self._json(403, {"error": "forbidden"})
|
|
122
|
+
if not target.is_file() or target.stat().st_size > _MAX_DOWNLOAD:
|
|
123
|
+
return self._json(404, {"error": "not found"})
|
|
124
|
+
data = target.read_bytes()
|
|
125
|
+
self.send_response(200)
|
|
126
|
+
self.send_header("Content-Type", mimetypes.guess_type(target.name)[0] or "application/octet-stream")
|
|
127
|
+
self.send_header("Content-Length", str(len(data)))
|
|
128
|
+
self.send_header("Content-Disposition", f'attachment; filename="{quote(target.name)}"')
|
|
129
|
+
self.send_header("X-Content-Type-Options", "nosniff")
|
|
130
|
+
self._cors()
|
|
131
|
+
self.end_headers()
|
|
132
|
+
self.wfile.write(data)
|
|
133
|
+
|
|
134
|
+
def do_POST(self) -> None:
|
|
135
|
+
url = urlparse(self.path)
|
|
136
|
+
if url.path != "/chat":
|
|
137
|
+
return self._json(404, {"error": "not found"})
|
|
138
|
+
if not self._authed(parse_qs(url.query)):
|
|
139
|
+
return self._json(401, {"error": "unauthorized"})
|
|
140
|
+
try:
|
|
141
|
+
length = int(self.headers.get("Content-Length", "0"))
|
|
142
|
+
except ValueError:
|
|
143
|
+
return self._json(400, {"error": "bad Content-Length"})
|
|
144
|
+
if length <= 0 or length > _MAX_BODY:
|
|
145
|
+
return self._json(413 if length > _MAX_BODY else 400, {"error": "body missing or too large"})
|
|
146
|
+
try:
|
|
147
|
+
payload = json.loads(self.rfile.read(length))
|
|
148
|
+
message, sid = payload["message"], payload.get("session", "default")
|
|
149
|
+
if not isinstance(message, str) or not message.strip() or len(message) > _MAX_MESSAGE:
|
|
150
|
+
raise ValueError("message")
|
|
151
|
+
if not isinstance(sid, str) or not _SESSION.fullmatch(sid):
|
|
152
|
+
raise ValueError("session")
|
|
153
|
+
except (ValueError, KeyError, TypeError):
|
|
154
|
+
return self._json(400, {"error": 'expected JSON {"message": "...", "session": "id"}'})
|
|
155
|
+
agent = sessions.get(sid)
|
|
156
|
+
root = str(agent.policy.workdir)
|
|
157
|
+
before = _snapshot(root)
|
|
158
|
+
try:
|
|
159
|
+
reply = agent.chat(message)
|
|
160
|
+
except LLMError as e:
|
|
161
|
+
return self._json(502, {"error": str(e)})
|
|
162
|
+
except Exception as e: # noqa: BLE001 - never leak internals to the client
|
|
163
|
+
print(f"botkit server error: {type(e).__name__}: {e}", file=sys.stderr)
|
|
164
|
+
return self._json(500, {"error": "internal error"})
|
|
165
|
+
after = _snapshot(root)
|
|
166
|
+
changed = [n for n, sig in after.items() if before.get(n) != sig][:50]
|
|
167
|
+
files = [{"name": n, "url": f"/files/{sid}/{quote(n)}"} for n in changed]
|
|
168
|
+
self._json(200, {"reply": reply, "files": files})
|
|
169
|
+
|
|
170
|
+
return Handler
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def serve(
|
|
174
|
+
agent: Any,
|
|
175
|
+
host: str = "127.0.0.1",
|
|
176
|
+
port: int = 8000,
|
|
177
|
+
token: str | None = None,
|
|
178
|
+
cors_origins: list[str] | None = None,
|
|
179
|
+
max_sessions: int = 200,
|
|
180
|
+
block: bool = True,
|
|
181
|
+
) -> ThreadingHTTPServer:
|
|
182
|
+
"""Start the HTTP API. With block=False the server runs in a background thread and is returned
|
|
183
|
+
(call .shutdown() to stop it)."""
|
|
184
|
+
if token is None and host not in _LOCAL:
|
|
185
|
+
raise BotError("Refusing to listen on a non-local address without token=... (anyone could use your bot).")
|
|
186
|
+
httpd = ThreadingHTTPServer((host, port), _make_handler(_Sessions(agent, max_sessions), token, cors_origins))
|
|
187
|
+
httpd.daemon_threads = True
|
|
188
|
+
if not block:
|
|
189
|
+
threading.Thread(target=httpd.serve_forever, daemon=True).start()
|
|
190
|
+
return httpd
|
|
191
|
+
print(f"botkit listening on http://{host}:{httpd.server_address[1]} (POST /chat)")
|
|
192
|
+
try:
|
|
193
|
+
httpd.serve_forever()
|
|
194
|
+
except KeyboardInterrupt:
|
|
195
|
+
pass
|
|
196
|
+
finally:
|
|
197
|
+
httpd.server_close()
|
|
198
|
+
agent.close()
|
|
199
|
+
return httpd
|