techtide-botkit 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- botkit/__init__.py +24 -0
- botkit/agent.py +449 -0
- botkit/errors.py +6 -0
- botkit/llm.py +111 -0
- botkit/mcp.py +268 -0
- botkit/memory/__init__.py +9 -0
- botkit/memory/base.py +56 -0
- botkit/memory/memory.py +109 -0
- botkit/memory/sqlite.py +115 -0
- botkit/memory/supabase.py +188 -0
- botkit/sandbox.py +139 -0
- botkit/security.py +98 -0
- botkit/server.py +199 -0
- botkit/skill.py +137 -0
- botkit/skills/coding/SKILL.md +10 -0
- botkit/skills/data-analysis/SKILL.md +9 -0
- botkit/skills/debugging/SKILL.md +10 -0
- botkit/skills/planning/SKILL.md +10 -0
- botkit/skills/presentations/SKILL.md +10 -0
- botkit/skills/research/SKILL.md +10 -0
- botkit/skills/spreadsheets/SKILL.md +10 -0
- botkit/skills/summarizing/SKILL.md +10 -0
- botkit/skills/verify-work/SKILL.md +10 -0
- botkit/skills/writing-documents/SKILL.md +10 -0
- botkit/tool.py +198 -0
- botkit/tools/__init__.py +42 -0
- botkit/tools/docs.py +195 -0
- botkit/tools/files.py +125 -0
- botkit/tools/shell.py +23 -0
- botkit/tools/util.py +65 -0
- botkit/tools/web.py +121 -0
- techtide_botkit-0.3.0.dist-info/METADATA +196 -0
- techtide_botkit-0.3.0.dist-info/RECORD +36 -0
- techtide_botkit-0.3.0.dist-info/WHEEL +5 -0
- techtide_botkit-0.3.0.dist-info/licenses/LICENSE +21 -0
- techtide_botkit-0.3.0.dist-info/top_level.txt +1 -0
botkit/tools/docs.py
ADDED
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
"""Document tools (3): create_docx, create_xlsx, create_pptx. Needs: pip install 'botkit[docs]'."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import re
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
from ..security import Policy
|
|
9
|
+
from ..tool import Tool, tool
|
|
10
|
+
|
|
11
|
+
OFFICE_EXT = {".docx", ".xlsx", ".pptx"}
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _need(pip_name: str) -> ImportError:
|
|
15
|
+
return ImportError(f"This tool needs '{pip_name}'. Install it with: pip install 'botkit[docs]'")
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
# ---------------------------------------------------------------- reading
|
|
19
|
+
def read_office(p: Path) -> str:
|
|
20
|
+
ext = p.suffix.lower()
|
|
21
|
+
if ext == ".docx":
|
|
22
|
+
try:
|
|
23
|
+
import docx
|
|
24
|
+
except ImportError:
|
|
25
|
+
raise _need("python-docx") from None
|
|
26
|
+
d = docx.Document(str(p))
|
|
27
|
+
parts = [para.text for para in d.paragraphs]
|
|
28
|
+
for table in d.tables:
|
|
29
|
+
parts += [" | ".join(c.text for c in row.cells) for row in table.rows]
|
|
30
|
+
return "\n".join(parts)
|
|
31
|
+
if ext == ".xlsx":
|
|
32
|
+
try:
|
|
33
|
+
import openpyxl
|
|
34
|
+
except ImportError:
|
|
35
|
+
raise _need("openpyxl") from None
|
|
36
|
+
wb = openpyxl.load_workbook(str(p))
|
|
37
|
+
out = []
|
|
38
|
+
for ws in wb.worksheets:
|
|
39
|
+
out.append(f"## {ws.title}")
|
|
40
|
+
for row in ws.iter_rows(max_row=500, values_only=True):
|
|
41
|
+
out.append(" | ".join("" if c is None else str(c) for c in row))
|
|
42
|
+
return "\n".join(out)
|
|
43
|
+
try:
|
|
44
|
+
from pptx import Presentation
|
|
45
|
+
except ImportError:
|
|
46
|
+
raise _need("python-pptx") from None
|
|
47
|
+
out = []
|
|
48
|
+
for i, slide in enumerate(Presentation(str(p)).slides, 1):
|
|
49
|
+
out.append(f"## Slide {i}")
|
|
50
|
+
for shape in slide.shapes:
|
|
51
|
+
if shape.has_text_frame:
|
|
52
|
+
out.append(shape.text_frame.text)
|
|
53
|
+
return "\n".join(out)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
# ---------------------------------------------------------------- docx content
|
|
57
|
+
def _runs(paragraph: Any, text: str) -> None:
|
|
58
|
+
for i, chunk in enumerate(re.split(r"\*\*(.+?)\*\*", text)):
|
|
59
|
+
if chunk:
|
|
60
|
+
paragraph.add_run(chunk).bold = i % 2 == 1
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _fill_docx(doc: Any, content: str) -> None:
|
|
64
|
+
table_rows: list[list[str]] = []
|
|
65
|
+
|
|
66
|
+
def flush_table() -> None:
|
|
67
|
+
if table_rows:
|
|
68
|
+
cols = max(len(r) for r in table_rows)
|
|
69
|
+
table = doc.add_table(rows=len(table_rows), cols=cols)
|
|
70
|
+
table.style = "Table Grid"
|
|
71
|
+
for r, row in enumerate(table_rows):
|
|
72
|
+
for c, cell in enumerate(row):
|
|
73
|
+
table.cell(r, c).text = cell
|
|
74
|
+
table_rows.clear()
|
|
75
|
+
|
|
76
|
+
for line in content.splitlines():
|
|
77
|
+
s = line.rstrip()
|
|
78
|
+
if s.strip().startswith("|"):
|
|
79
|
+
cells = [c.strip() for c in s.strip().strip("|").split("|")]
|
|
80
|
+
if not all(re.fullmatch(r":?-{2,}:?", c) for c in cells):
|
|
81
|
+
table_rows.append(cells)
|
|
82
|
+
continue
|
|
83
|
+
flush_table()
|
|
84
|
+
if not s.strip():
|
|
85
|
+
continue
|
|
86
|
+
if m := re.match(r"^(#{1,4})\s+(.*)", s):
|
|
87
|
+
doc.add_heading(m.group(2), level=len(m.group(1)))
|
|
88
|
+
elif m := re.match(r"^\s*[-*]\s+(.*)", s):
|
|
89
|
+
_runs(doc.add_paragraph(style="List Bullet"), m.group(1))
|
|
90
|
+
elif m := re.match(r"^\s*\d+[.)]\s+(.*)", s):
|
|
91
|
+
_runs(doc.add_paragraph(style="List Number"), m.group(1))
|
|
92
|
+
else:
|
|
93
|
+
_runs(doc.add_paragraph(), s)
|
|
94
|
+
flush_table()
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def make(policy: Policy, sandbox=None) -> list[Tool]:
|
|
98
|
+
def target(path: str, ext: str) -> Path:
|
|
99
|
+
return policy.resolve(path if path.lower().endswith(ext) else path + ext)
|
|
100
|
+
|
|
101
|
+
def rel(p: Path) -> Path:
|
|
102
|
+
return p.relative_to(policy.workdir.resolve())
|
|
103
|
+
|
|
104
|
+
@tool
|
|
105
|
+
def create_docx(path: str, title: str, content: str) -> str:
|
|
106
|
+
"""Create a Word document. Content uses simple markdown: # headings, - bullets, 1. numbered, **bold**, | tables |.
|
|
107
|
+
|
|
108
|
+
Args:
|
|
109
|
+
path: Output path relative to the workspace, e.g. "report.docx".
|
|
110
|
+
title: Document title.
|
|
111
|
+
content: The body in simple markdown.
|
|
112
|
+
"""
|
|
113
|
+
try:
|
|
114
|
+
import docx
|
|
115
|
+
except ImportError:
|
|
116
|
+
raise _need("python-docx") from None
|
|
117
|
+
p = target(path, ".docx")
|
|
118
|
+
doc = docx.Document()
|
|
119
|
+
doc.add_heading(title, level=0)
|
|
120
|
+
_fill_docx(doc, content)
|
|
121
|
+
p.parent.mkdir(parents=True, exist_ok=True)
|
|
122
|
+
doc.save(str(p))
|
|
123
|
+
return f"Created {rel(p)}"
|
|
124
|
+
|
|
125
|
+
@tool
|
|
126
|
+
def create_xlsx(path: str, sheets: dict) -> str:
|
|
127
|
+
"""Create an Excel workbook. Strings starting with "=" become formulas; the first row of each sheet is bold.
|
|
128
|
+
|
|
129
|
+
Args:
|
|
130
|
+
path: Output path relative to the workspace, e.g. "budget.xlsx".
|
|
131
|
+
sheets: Mapping of sheet name to a list of rows, each row a list of cell values.
|
|
132
|
+
"""
|
|
133
|
+
try:
|
|
134
|
+
import openpyxl
|
|
135
|
+
from openpyxl.styles import Font
|
|
136
|
+
except ImportError:
|
|
137
|
+
raise _need("openpyxl") from None
|
|
138
|
+
if not isinstance(sheets, dict) or not sheets:
|
|
139
|
+
raise ValueError("sheets must be a non-empty object: {sheet name: [[row], [row]]}")
|
|
140
|
+
p = target(path, ".xlsx")
|
|
141
|
+
wb = openpyxl.Workbook()
|
|
142
|
+
wb.remove(wb.active)
|
|
143
|
+
for name, rows in sheets.items():
|
|
144
|
+
if not isinstance(rows, list) or not all(isinstance(r, (list, tuple)) for r in rows):
|
|
145
|
+
raise ValueError(f"sheet '{name}' must be a list of rows (lists)")
|
|
146
|
+
ws = wb.create_sheet(title=(str(name) or "Sheet")[:31])
|
|
147
|
+
for row in rows:
|
|
148
|
+
ws.append(list(row))
|
|
149
|
+
if rows:
|
|
150
|
+
for cell in ws[1]:
|
|
151
|
+
cell.font = Font(bold=True)
|
|
152
|
+
for col in ws.columns:
|
|
153
|
+
width = max((len(str(c.value)) for c in col if c.value is not None), default=8)
|
|
154
|
+
ws.column_dimensions[col[0].column_letter].width = min(width + 2, 50)
|
|
155
|
+
p.parent.mkdir(parents=True, exist_ok=True)
|
|
156
|
+
wb.save(str(p))
|
|
157
|
+
return f"Created {rel(p)}"
|
|
158
|
+
|
|
159
|
+
@tool
|
|
160
|
+
def create_pptx(path: str, slides: list[dict]) -> str:
|
|
161
|
+
"""Create a PowerPoint deck. Each slide is an object with "title", optional "bullets" (list of strings),
|
|
162
|
+
optional "subtitle" (for a title slide) and optional "notes" (speaker notes).
|
|
163
|
+
|
|
164
|
+
Args:
|
|
165
|
+
path: Output path relative to the workspace, e.g. "deck.pptx".
|
|
166
|
+
slides: The slides, in order.
|
|
167
|
+
"""
|
|
168
|
+
try:
|
|
169
|
+
from pptx import Presentation
|
|
170
|
+
except ImportError:
|
|
171
|
+
raise _need("python-pptx") from None
|
|
172
|
+
if not slides:
|
|
173
|
+
raise ValueError("slides must not be empty")
|
|
174
|
+
p = target(path, ".pptx")
|
|
175
|
+
prs = Presentation()
|
|
176
|
+
for s in slides:
|
|
177
|
+
bullets = [str(b) for b in s.get("bullets") or []]
|
|
178
|
+
if s.get("subtitle") and not bullets:
|
|
179
|
+
slide = prs.slides.add_slide(prs.slide_layouts[0])
|
|
180
|
+
slide.shapes.title.text = str(s.get("title", ""))
|
|
181
|
+
slide.placeholders[1].text = str(s["subtitle"])
|
|
182
|
+
else:
|
|
183
|
+
slide = prs.slides.add_slide(prs.slide_layouts[1])
|
|
184
|
+
slide.shapes.title.text = str(s.get("title", ""))
|
|
185
|
+
frame = slide.placeholders[1].text_frame
|
|
186
|
+
frame.text = bullets[0] if bullets else ""
|
|
187
|
+
for b in bullets[1:]:
|
|
188
|
+
frame.add_paragraph().text = b
|
|
189
|
+
if s.get("notes"):
|
|
190
|
+
slide.notes_slide.notes_text_frame.text = str(s["notes"])
|
|
191
|
+
p.parent.mkdir(parents=True, exist_ok=True)
|
|
192
|
+
prs.save(str(p))
|
|
193
|
+
return f"Created {rel(p)} ({len(slides)} slides)"
|
|
194
|
+
|
|
195
|
+
return [create_docx, create_xlsx, create_pptx]
|
botkit/tools/files.py
ADDED
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
"""File tools (5): list_files, read_file, write_file, edit_file, search_files. Confined to the workspace."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import fnmatch
|
|
5
|
+
import os
|
|
6
|
+
import re
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
from ..security import Policy
|
|
10
|
+
from ..tool import Tool, clip, tool
|
|
11
|
+
from .docs import OFFICE_EXT, read_office
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def make(policy: Policy, sandbox=None) -> list[Tool]:
|
|
15
|
+
@tool
|
|
16
|
+
def list_files(path: str = ".") -> str:
|
|
17
|
+
"""List files and folders in the workspace.
|
|
18
|
+
|
|
19
|
+
Args:
|
|
20
|
+
path: Folder relative to the workspace (default: the workspace root).
|
|
21
|
+
"""
|
|
22
|
+
p = policy.resolve(path)
|
|
23
|
+
if not p.is_dir():
|
|
24
|
+
raise NotADirectoryError(f"'{path}' is not a folder")
|
|
25
|
+
entries = sorted(p.iterdir(), key=lambda e: (not e.is_dir(), e.name.lower()))
|
|
26
|
+
lines = [f"{e.name}/" if e.is_dir() else f"{e.name} ({e.stat().st_size} bytes)" for e in entries[:200]]
|
|
27
|
+
return "\n".join(lines) or "(empty)"
|
|
28
|
+
|
|
29
|
+
@tool
|
|
30
|
+
def read_file(path: str, max_chars: int = 20000) -> str:
|
|
31
|
+
"""Read a file from the workspace. Works for text files and .docx/.xlsx/.pptx documents.
|
|
32
|
+
|
|
33
|
+
Args:
|
|
34
|
+
path: File path relative to the workspace.
|
|
35
|
+
max_chars: Maximum characters to return.
|
|
36
|
+
"""
|
|
37
|
+
p = policy.resolve(path)
|
|
38
|
+
if not p.is_file():
|
|
39
|
+
raise FileNotFoundError(f"no such file: {path}")
|
|
40
|
+
if p.stat().st_size > policy.max_file_bytes:
|
|
41
|
+
raise ValueError("file is too large to read")
|
|
42
|
+
if p.suffix.lower() in OFFICE_EXT:
|
|
43
|
+
return clip(read_office(p), max_chars)
|
|
44
|
+
raw = p.read_bytes()
|
|
45
|
+
if b"\0" in raw[:2048]:
|
|
46
|
+
return f"[binary file, {len(raw)} bytes]"
|
|
47
|
+
return clip(raw.decode("utf-8", errors="replace"), max_chars)
|
|
48
|
+
|
|
49
|
+
@tool
|
|
50
|
+
def write_file(path: str, content: str) -> str:
|
|
51
|
+
"""Create or overwrite a text file in the workspace.
|
|
52
|
+
|
|
53
|
+
Args:
|
|
54
|
+
path: File path relative to the workspace.
|
|
55
|
+
content: The full file content.
|
|
56
|
+
"""
|
|
57
|
+
data = content.encode("utf-8")
|
|
58
|
+
if len(data) > policy.max_file_bytes:
|
|
59
|
+
raise ValueError("content is too large")
|
|
60
|
+
p = policy.resolve(path)
|
|
61
|
+
p.parent.mkdir(parents=True, exist_ok=True)
|
|
62
|
+
p.write_bytes(data)
|
|
63
|
+
return f"Wrote {len(data)} bytes to {path}"
|
|
64
|
+
|
|
65
|
+
@tool
|
|
66
|
+
def edit_file(path: str, old: str, new: str) -> str:
|
|
67
|
+
"""Replace one exact piece of text in a workspace file (the text must appear exactly once).
|
|
68
|
+
|
|
69
|
+
Args:
|
|
70
|
+
path: File path relative to the workspace.
|
|
71
|
+
old: The exact text to replace.
|
|
72
|
+
new: The replacement text.
|
|
73
|
+
"""
|
|
74
|
+
p = policy.resolve(path)
|
|
75
|
+
text = p.read_text(encoding="utf-8")
|
|
76
|
+
count = text.count(old)
|
|
77
|
+
if count != 1:
|
|
78
|
+
raise ValueError(f"'old' must match exactly once, found {count} matches")
|
|
79
|
+
p.write_text(text.replace(old, new), encoding="utf-8")
|
|
80
|
+
return f"Edited {path}"
|
|
81
|
+
|
|
82
|
+
@tool
|
|
83
|
+
def search_files(pattern: str, path: str = ".", glob: str = "*", max_results: int = 50) -> str:
|
|
84
|
+
"""Search text files in the workspace for a regular expression (like grep). Returns file:line: text.
|
|
85
|
+
|
|
86
|
+
Args:
|
|
87
|
+
pattern: Regular expression to look for (case-insensitive).
|
|
88
|
+
path: Folder or file to search, relative to the workspace.
|
|
89
|
+
glob: Only search files whose name matches this pattern, e.g. "*.py".
|
|
90
|
+
max_results: Maximum number of matching lines to return.
|
|
91
|
+
"""
|
|
92
|
+
try:
|
|
93
|
+
rx = re.compile(pattern, re.I)
|
|
94
|
+
except re.error as e:
|
|
95
|
+
raise ValueError(f"invalid regular expression: {e}") from None
|
|
96
|
+
base = policy.workdir.resolve()
|
|
97
|
+
start = policy.resolve(path)
|
|
98
|
+
if start.is_file():
|
|
99
|
+
candidates = [start]
|
|
100
|
+
else:
|
|
101
|
+
candidates = []
|
|
102
|
+
for dirpath, dirs, names in os.walk(start):
|
|
103
|
+
dirs[:] = sorted(d for d in dirs if not d.startswith("."))
|
|
104
|
+
candidates += [Path(dirpath) / n for n in sorted(names) if fnmatch.fnmatch(n, glob)]
|
|
105
|
+
if len(candidates) > 5000:
|
|
106
|
+
break
|
|
107
|
+
hits: list[str] = []
|
|
108
|
+
for file in candidates:
|
|
109
|
+
try:
|
|
110
|
+
real = policy.resolve(str(file.relative_to(base))) # symlinks pointing outside are skipped
|
|
111
|
+
if not real.is_file() or real.stat().st_size > 1_000_000:
|
|
112
|
+
continue
|
|
113
|
+
raw = real.read_bytes()
|
|
114
|
+
except (PermissionError, ValueError, OSError):
|
|
115
|
+
continue
|
|
116
|
+
if b"\0" in raw[:2048]:
|
|
117
|
+
continue
|
|
118
|
+
for lineno, line in enumerate(raw.decode("utf-8", errors="replace").splitlines(), 1):
|
|
119
|
+
if rx.search(line):
|
|
120
|
+
hits.append(f"{file.relative_to(base)}:{lineno}: {line.strip()[:200]}")
|
|
121
|
+
if len(hits) >= max_results:
|
|
122
|
+
return "\n".join(hits) + f"\n[stopped after {max_results} matches]"
|
|
123
|
+
return "\n".join(hits) or "No matches."
|
|
124
|
+
|
|
125
|
+
return [list_files, read_file, write_file, edit_file, search_files]
|
botkit/tools/shell.py
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
"""Shell tool (1): run_command. Only exists when you pick an environment with Agent(shell=...)."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
from ..security import Policy
|
|
5
|
+
from ..tool import Tool, tool
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def make(policy: Policy, sandbox=None) -> list[Tool]:
|
|
9
|
+
if sandbox is None:
|
|
10
|
+
return []
|
|
11
|
+
|
|
12
|
+
@tool(sensitive=True)
|
|
13
|
+
def run_command(command: str, timeout: int = 60) -> str:
|
|
14
|
+
"""Run a shell command in the sandbox. The working directory is the workspace.
|
|
15
|
+
|
|
16
|
+
Args:
|
|
17
|
+
command: The shell command.
|
|
18
|
+
timeout: Seconds before the command is killed.
|
|
19
|
+
"""
|
|
20
|
+
code, output = sandbox.run(command, min(max(int(timeout), 1), policy.shell_timeout_max))
|
|
21
|
+
return f"exit code: {code}\n{output}"
|
|
22
|
+
|
|
23
|
+
return [run_command]
|
botkit/tools/util.py
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
"""Utility tool (1): calculate. Exact, safe arithmetic (no eval of arbitrary code)."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import ast
|
|
5
|
+
import math
|
|
6
|
+
import operator
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
from ..security import Policy
|
|
10
|
+
from ..tool import Tool, tool
|
|
11
|
+
|
|
12
|
+
_OPS = {
|
|
13
|
+
ast.Add: operator.add, ast.Sub: operator.sub, ast.Mult: operator.mul, ast.Div: operator.truediv,
|
|
14
|
+
ast.FloorDiv: operator.floordiv, ast.Mod: operator.mod, ast.Pow: operator.pow,
|
|
15
|
+
}
|
|
16
|
+
_FUNCS = {
|
|
17
|
+
"sqrt": math.sqrt, "abs": abs, "round": round, "min": min, "max": max, "log": math.log, "log10": math.log10,
|
|
18
|
+
"sin": math.sin, "cos": math.cos, "tan": math.tan, "floor": math.floor, "ceil": math.ceil,
|
|
19
|
+
}
|
|
20
|
+
_CONSTS = {"pi": math.pi, "e": math.e}
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def safe_eval(expression: str) -> Any:
|
|
24
|
+
if len(expression) > 500:
|
|
25
|
+
raise ValueError("expression too long")
|
|
26
|
+
try:
|
|
27
|
+
tree = ast.parse(expression.strip(), mode="eval")
|
|
28
|
+
except SyntaxError:
|
|
29
|
+
raise ValueError("not a valid arithmetic expression") from None
|
|
30
|
+
|
|
31
|
+
def ev(node: ast.AST) -> Any:
|
|
32
|
+
if isinstance(node, ast.Expression):
|
|
33
|
+
return ev(node.body)
|
|
34
|
+
if isinstance(node, ast.Constant) and isinstance(node.value, (int, float)) and not isinstance(node.value, bool):
|
|
35
|
+
return node.value
|
|
36
|
+
if isinstance(node, ast.BinOp) and type(node.op) in _OPS:
|
|
37
|
+
left, right = ev(node.left), ev(node.right)
|
|
38
|
+
if isinstance(node.op, ast.Pow) and isinstance(left, int) and isinstance(right, int) and right > 0:
|
|
39
|
+
if max(left.bit_length(), 1) * right > 10_000:
|
|
40
|
+
raise ValueError("number too large")
|
|
41
|
+
return _OPS[type(node.op)](left, right)
|
|
42
|
+
if isinstance(node, ast.UnaryOp) and isinstance(node.op, (ast.USub, ast.UAdd)):
|
|
43
|
+
return -ev(node.operand) if isinstance(node.op, ast.USub) else ev(node.operand)
|
|
44
|
+
if isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and node.func.id in _FUNCS and not node.keywords:
|
|
45
|
+
if len(node.args) > 10:
|
|
46
|
+
raise ValueError("too many arguments")
|
|
47
|
+
return _FUNCS[node.func.id](*[ev(a) for a in node.args])
|
|
48
|
+
if isinstance(node, ast.Name) and node.id in _CONSTS:
|
|
49
|
+
return _CONSTS[node.id]
|
|
50
|
+
raise ValueError("unsupported expression (numbers, + - * / // % **, parentheses, sqrt/abs/round/min/max/log/sin/cos/tan/floor/ceil, pi, e)")
|
|
51
|
+
|
|
52
|
+
return ev(tree)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def make(policy: Policy, sandbox=None) -> list[Tool]:
|
|
56
|
+
@tool
|
|
57
|
+
def calculate(expression: str) -> str:
|
|
58
|
+
"""Evaluate an arithmetic expression exactly, e.g. "(1250 * 1.18) / 12" or "sqrt(2) * pi". Use this instead of doing maths in your head.
|
|
59
|
+
|
|
60
|
+
Args:
|
|
61
|
+
expression: The arithmetic expression.
|
|
62
|
+
"""
|
|
63
|
+
return str(safe_eval(expression))
|
|
64
|
+
|
|
65
|
+
return [calculate]
|
botkit/tools/web.py
ADDED
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
"""Web tools (2): fetch_url, web_search. Public internet only (private/internal addresses are blocked)."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import html
|
|
5
|
+
import ipaddress
|
|
6
|
+
import re
|
|
7
|
+
import socket
|
|
8
|
+
import urllib.error
|
|
9
|
+
import urllib.parse
|
|
10
|
+
import urllib.request
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
13
|
+
from ..security import Policy
|
|
14
|
+
from ..tool import Tool, clip, tool
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _check_public_host(host: str) -> None:
|
|
18
|
+
"""Refuse hosts that resolve to private/loopback/link-local addresses (SSRF protection)."""
|
|
19
|
+
for info in socket.getaddrinfo(host, None):
|
|
20
|
+
ip = ipaddress.ip_address(info[4][0])
|
|
21
|
+
ip = getattr(ip, "ipv4_mapped", None) or ip
|
|
22
|
+
if ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_reserved or ip.is_multicast or ip.is_unspecified:
|
|
23
|
+
raise PermissionError(f"blocked: '{host}' is not a public internet address")
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class _NoRedirect(urllib.request.HTTPRedirectHandler):
|
|
27
|
+
def redirect_request(self, *args: Any, **kwargs: Any) -> None: # we follow redirects ourselves, re-checking each hop
|
|
28
|
+
return None
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _fetch(url: str, max_bytes: int = 1_000_000) -> tuple[str, str]:
|
|
32
|
+
opener = urllib.request.build_opener(_NoRedirect)
|
|
33
|
+
for _ in range(5):
|
|
34
|
+
parsed = urllib.parse.urlparse(url)
|
|
35
|
+
if parsed.scheme not in ("http", "https") or not parsed.hostname:
|
|
36
|
+
raise PermissionError("only http(s) URLs are allowed")
|
|
37
|
+
_check_public_host(parsed.hostname)
|
|
38
|
+
req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0 (compatible; botkit/0.3)"})
|
|
39
|
+
try:
|
|
40
|
+
resp = opener.open(req, timeout=15)
|
|
41
|
+
except urllib.error.HTTPError as e:
|
|
42
|
+
location = e.headers.get("Location") if e.headers else None
|
|
43
|
+
if e.code in (301, 302, 303, 307, 308) and location:
|
|
44
|
+
url = urllib.parse.urljoin(url, location)
|
|
45
|
+
continue
|
|
46
|
+
raise
|
|
47
|
+
with resp:
|
|
48
|
+
kind = resp.headers.get_content_type()
|
|
49
|
+
if not (kind.startswith("text/") or kind.endswith(("json", "xml"))):
|
|
50
|
+
return kind, f"[{kind} content, not shown]"
|
|
51
|
+
charset = resp.headers.get_content_charset() or "utf-8"
|
|
52
|
+
return kind, resp.read(max_bytes).decode(charset, errors="replace")
|
|
53
|
+
raise ValueError("too many redirects")
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _html_to_text(raw: str) -> str:
|
|
57
|
+
raw = re.sub(r"(?is)<(script|style|noscript)\b.*?</\1>", " ", raw)
|
|
58
|
+
raw = re.sub(r"(?s)<!--.*?-->", " ", raw)
|
|
59
|
+
raw = re.sub(r"(?i)<(br|/p|/div|/li|/h\d|/tr)\b[^>]*>", "\n", raw)
|
|
60
|
+
text = html.unescape(re.sub(r"<[^>]+>", " ", raw))
|
|
61
|
+
return re.sub(r"\n\s*\n+", "\n\n", re.sub(r"[ \t]+", " ", text)).strip()
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
_RESULT_A = re.compile(r'<a\s[^>]*class="[^"]*\bresult__a\b[^"]*"[^>]*>(.*?)</a>', re.S)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def parse_search_results(page: str, limit: int = 8) -> list[dict[str, str]]:
|
|
68
|
+
"""Parse DuckDuckGo's HTML results page into [{title, url, snippet}]."""
|
|
69
|
+
matches = list(_RESULT_A.finditer(page))
|
|
70
|
+
results = []
|
|
71
|
+
for i, m in enumerate(matches[:limit]):
|
|
72
|
+
href_m = re.search(r'href="([^"]*)"', m.group(0))
|
|
73
|
+
if not href_m:
|
|
74
|
+
continue
|
|
75
|
+
href = html.unescape(href_m.group(1))
|
|
76
|
+
if "uddg=" in href: # DuckDuckGo wraps the real link
|
|
77
|
+
query = urllib.parse.urlparse(href if href.startswith("http") else "https:" + href).query
|
|
78
|
+
href = urllib.parse.parse_qs(query).get("uddg", [href])[0]
|
|
79
|
+
end = matches[i + 1].start() if i + 1 < len(matches) else len(page)
|
|
80
|
+
sn = re.search(r'class="[^"]*result__snippet[^"]*"[^>]*>(.*?)</(?:a|td|div|span)>', page[m.end():end], re.S)
|
|
81
|
+
results.append(
|
|
82
|
+
{"title": _html_to_text(m.group(1)), "url": href, "snippet": _html_to_text(sn.group(1)) if sn else ""}
|
|
83
|
+
)
|
|
84
|
+
return results
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def make(policy: Policy, sandbox=None) -> list[Tool]:
|
|
88
|
+
if not policy.allow_network:
|
|
89
|
+
return []
|
|
90
|
+
|
|
91
|
+
@tool
|
|
92
|
+
def fetch_url(url: str, max_chars: int = 15000) -> str:
|
|
93
|
+
"""Download a public web page or API response and return its text. Private/internal addresses are blocked.
|
|
94
|
+
|
|
95
|
+
Args:
|
|
96
|
+
url: A public http(s) URL.
|
|
97
|
+
max_chars: Maximum characters to return.
|
|
98
|
+
"""
|
|
99
|
+
kind, body = _fetch(url)
|
|
100
|
+
if kind == "text/html":
|
|
101
|
+
body = _html_to_text(body)
|
|
102
|
+
return clip(body, max_chars)
|
|
103
|
+
|
|
104
|
+
@tool
|
|
105
|
+
def web_search(query: str, max_results: int = 6) -> str:
|
|
106
|
+
"""Search the web (DuckDuckGo) and return titles, URLs and snippets. Use fetch_url to read a result.
|
|
107
|
+
|
|
108
|
+
Args:
|
|
109
|
+
query: What to search for.
|
|
110
|
+
max_results: How many results to return (1-10).
|
|
111
|
+
"""
|
|
112
|
+
url = "https://html.duckduckgo.com/html/?q=" + urllib.parse.quote(query)
|
|
113
|
+
_, page = _fetch(url)
|
|
114
|
+
results = parse_search_results(page, max(1, min(int(max_results), 10)))
|
|
115
|
+
if not results:
|
|
116
|
+
return "No results (the search provider may be blocking automated requests). Try fetch_url on a known site."
|
|
117
|
+
return "\n\n".join(
|
|
118
|
+
f"{i}. {r['title']}\n {r['url']}\n {r['snippet']}".rstrip() for i, r in enumerate(results, 1)
|
|
119
|
+
)
|
|
120
|
+
|
|
121
|
+
return [fetch_url, web_search]
|