atomzero 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,78 @@
1
+ """Store manually supplied teacher answers as unverified RAG data.
2
+
3
+ This module is shared by the HTTP server and the reviewed research queue so both
4
+ paths use the same provenance, limits and `verified=false` marking.
5
+ """
6
+ from __future__ import annotations
7
+
8
+ import hashlib
9
+ import re
10
+ import urllib.parse
11
+
12
+ MAX_TEACHER_BYTES = 200_000
13
+ MAX_TEACHER_TOPIC = 300
14
+ MAX_TEACHER_LABEL = 100
15
+ MAX_SOURCE_URL = 2048
16
+
17
+
18
+ class TeacherNoteTooLarge(ValueError):
19
+ pass
20
+
21
+
22
+ def add_teacher_note(docs, teacher, model, topic, answer,
23
+ source_url: str = "") -> dict:
24
+ """Validate and store a teacher answer with derived/unverified metadata."""
25
+ if docs is None:
26
+ raise ValueError("no document memory configured")
27
+ teacher = str(teacher or "manual").strip() or "manual"
28
+ model = str(model or "").strip()
29
+ topic = str(topic or "").strip()
30
+ answer = str(answer or "").strip()
31
+ source_url = str(source_url or "").strip()
32
+ for label, value, limit in (
33
+ ("teacher", teacher, MAX_TEACHER_LABEL),
34
+ ("model", model, MAX_TEACHER_LABEL),
35
+ ("topic", topic, MAX_TEACHER_TOPIC)):
36
+ if len(value) > limit:
37
+ raise ValueError(f"{label} is too long (max {limit} chars)")
38
+ if any(ord(ch) < 32 for ch in value):
39
+ raise ValueError(f"{label} contains control characters")
40
+ if not answer:
41
+ raise ValueError("answer is required")
42
+ if source_url:
43
+ if len(source_url) > MAX_SOURCE_URL:
44
+ raise ValueError(
45
+ f"source_url is too long (max {MAX_SOURCE_URL} chars)")
46
+ parsed = urllib.parse.urlparse(source_url)
47
+ if parsed.scheme not in ("http", "https") or not parsed.hostname:
48
+ raise ValueError("source_url must be an http(s) URL")
49
+ text = f"Topic: {topic}\n\n{answer}" if topic else answer
50
+ raw = text.encode("utf-8")
51
+ if len(raw) > MAX_TEACHER_BYTES:
52
+ raise TeacherNoteTooLarge(
53
+ f"teacher note is too large (max {MAX_TEACHER_BYTES} bytes)")
54
+ safe = re.sub(r"[^A-Za-z0-9_.-]+", "-", teacher).strip("-")[:48]
55
+ digest = hashlib.sha256(raw).hexdigest()
56
+ source = f"teacher://{safe or 'manual'}/{digest[:16]}"
57
+ title = (f"Teacher note: {topic}" if topic
58
+ else f"Teacher note ({teacher})")
59
+ chunks = docs.add_text(source, text, title=title, meta={
60
+ "kind": "teacher",
61
+ "source_url": source_url,
62
+ "teacher": teacher,
63
+ "model": model,
64
+ "verified": False,
65
+ })
66
+ return {
67
+ "source": source,
68
+ "title": title,
69
+ "chunks": chunks,
70
+ "bytes": len(raw),
71
+ "sha256": digest,
72
+ "kind": "teacher",
73
+ "verified": False,
74
+ "source_url": source_url,
75
+ "teacher": teacher,
76
+ "model": model,
77
+ "note": "teacher-derived and unverified",
78
+ }
@@ -0,0 +1,104 @@
1
+ """Optional OpenAI-compatible teacher provider.
2
+
3
+ Point ``base_url`` at a local Ollama OpenAI-compatible endpoint or another
4
+ compatible server. The provider is only called by an explicit request; it is
5
+ never used automatically and its output is stored as unverified teacher data.
6
+ """
7
+ from __future__ import annotations
8
+
9
+ import json
10
+ import urllib.error
11
+ import urllib.parse
12
+ import urllib.request
13
+
14
+ USER_AGENT = "streamft-local-assistant/0.1"
15
+ MAX_RESPONSE_BYTES = 2_000_000
16
+
17
+
18
+ class TeacherProviderError(RuntimeError):
19
+ pass
20
+
21
+
22
+ class TeacherProvider:
23
+ def __init__(self, base_url: str, model: str, api_key: str = "",
24
+ timeout: int = 60, label: str = "openai-compatible"):
25
+ base_url = str(base_url or "").strip()
26
+ model = str(model or "").strip()
27
+ if not base_url or not model:
28
+ raise TeacherProviderError(
29
+ "teacher base URL and model are required")
30
+ parsed = urllib.parse.urlparse(base_url)
31
+ if parsed.scheme not in ("http", "https") or not parsed.hostname:
32
+ raise TeacherProviderError("teacher URL must be an http(s) URL")
33
+ self.base_url = base_url.rstrip("/")
34
+ self.model = model
35
+ self.api_key = str(api_key or "")
36
+ try:
37
+ self.timeout = max(1, min(int(timeout), 300))
38
+ except (TypeError, ValueError) as exc:
39
+ raise TeacherProviderError(
40
+ "teacher timeout must be an integer") from exc
41
+ self.label = str(label or "openai-compatible")[:100]
42
+ self.host = parsed.netloc
43
+ self.endpoint = (
44
+ self.base_url if self.base_url.endswith("/chat/completions")
45
+ else self.base_url + "/chat/completions")
46
+
47
+ def ask(self, prompt: str, system: str = "",
48
+ max_tokens: int = 512) -> str:
49
+ prompt = str(prompt or "").strip()
50
+ if not prompt:
51
+ raise TeacherProviderError("prompt is required")
52
+ try:
53
+ max_tokens = max(1, min(int(max_tokens), 2048))
54
+ except (TypeError, ValueError) as exc:
55
+ raise TeacherProviderError("max_tokens must be an integer") from exc
56
+ messages = []
57
+ if system:
58
+ messages.append({"role": "system", "content": str(system)})
59
+ messages.append({"role": "user", "content": prompt})
60
+ body = {
61
+ "model": self.model,
62
+ "messages": messages,
63
+ "temperature": 0.2,
64
+ "max_tokens": max_tokens,
65
+ }
66
+ headers = {
67
+ "Content-Type": "application/json",
68
+ "User-Agent": USER_AGENT,
69
+ }
70
+ if self.api_key:
71
+ headers["Authorization"] = f"Bearer {self.api_key}"
72
+ request = urllib.request.Request(
73
+ self.endpoint, data=json.dumps(body).encode("utf-8"),
74
+ headers=headers, method="POST")
75
+ try:
76
+ with urllib.request.urlopen(
77
+ request, timeout=self.timeout) as response:
78
+ raw = response.read(MAX_RESPONSE_BYTES + 1)
79
+ except urllib.error.HTTPError as exc:
80
+ try:
81
+ detail = exc.read(500).decode("utf-8", errors="replace")
82
+ except Exception:
83
+ detail = ""
84
+ raise TeacherProviderError(
85
+ f"teacher provider HTTP {exc.code}: {detail}") from exc
86
+ except Exception as exc:
87
+ raise TeacherProviderError(
88
+ f"teacher provider request failed: {exc}") from exc
89
+ if len(raw) > MAX_RESPONSE_BYTES:
90
+ raise TeacherProviderError("teacher provider response is too large")
91
+ try:
92
+ data = json.loads(raw.decode("utf-8"))
93
+ except (UnicodeDecodeError, json.JSONDecodeError) as exc:
94
+ raise TeacherProviderError(
95
+ "teacher provider returned invalid JSON") from exc
96
+ try:
97
+ content = data["choices"][0]["message"]["content"]
98
+ except (KeyError, IndexError, TypeError) as exc:
99
+ raise TeacherProviderError(
100
+ "teacher provider response has no message content") from exc
101
+ content = str(content or "").strip()
102
+ if not content:
103
+ raise TeacherProviderError("teacher provider returned an empty answer")
104
+ return content
@@ -0,0 +1,207 @@
1
+ """Constrained tool-plan validation and execution.
2
+
3
+ This is an allowlisted executor, not free-form model planning. A caller (the
4
+ UI, a test, or a future provider-assisted planner) submits a small JSON plan;
5
+ this module rejects unknown tools/arguments, caps the plan at three steps and
6
+ requires explicit confirmation before any side effect.
7
+
8
+ The 49.2M core does not choose these steps. That boundary is deliberate: a
9
+ small model can propose a plan later, but it cannot bypass this validator.
10
+ """
11
+ from __future__ import annotations
12
+
13
+ from datetime import datetime, timezone
14
+
15
+ MAX_STEPS = 3
16
+
17
+ TOOL_SPECS = {
18
+ "time": {"side_effect": False, "args": ()},
19
+ "calculator": {"side_effect": False, "args": ("expression",)},
20
+ "search_docs": {"side_effect": False, "args": ("query", "k")},
21
+ "evidence": {"side_effect": False, "args": ("source", "page", "section")},
22
+ "task_list": {"side_effect": False, "args": ()},
23
+ "task_add": {"side_effect": True, "args": ("text",)},
24
+ "task_complete": {"side_effect": True, "args": ("id",)},
25
+ "research": {"side_effect": True, "args": ("query",)},
26
+ }
27
+
28
+
29
+ class ToolPlanError(ValueError):
30
+ """Raised for an invalid plan or an unsafe tool invocation."""
31
+
32
+
33
+ def _text(value, name: str, maximum: int, required: bool = True) -> str:
34
+ if not isinstance(value, str):
35
+ raise ToolPlanError(f"{name} must be a string")
36
+ value = value.strip()
37
+ if required and not value:
38
+ raise ToolPlanError(f"{name} is required")
39
+ if len(value) > maximum:
40
+ raise ToolPlanError(f"{name} is too long (max {maximum} chars)")
41
+ if any(ord(ch) < 32 and ch not in "\n\t" for ch in value):
42
+ raise ToolPlanError(f"{name} contains control characters")
43
+ return value
44
+
45
+
46
+ def _integer(value, name: str, minimum: int = 1,
47
+ maximum: int = 1_000_000) -> int:
48
+ if isinstance(value, bool) or not isinstance(value, int):
49
+ raise ToolPlanError(f"{name} must be an integer")
50
+ if value < minimum or value > maximum:
51
+ raise ToolPlanError(
52
+ f"{name} must be between {minimum} and {maximum}")
53
+ return value
54
+
55
+
56
+ def _validate_args(tool: str, raw) -> dict:
57
+ if raw is None:
58
+ raw = {}
59
+ if not isinstance(raw, dict):
60
+ raise ToolPlanError("args must be an object")
61
+ allowed = set(TOOL_SPECS[tool]["args"])
62
+ unknown = set(raw) - allowed
63
+ if unknown:
64
+ raise ToolPlanError(
65
+ f"unknown argument(s) for {tool}: {', '.join(sorted(unknown))}")
66
+ if tool == "calculator":
67
+ return {"expression": _text(raw.get("expression"), "expression", 80)}
68
+ if tool == "search_docs":
69
+ args = {"query": _text(raw.get("query"), "query", 500)}
70
+ if "k" in raw:
71
+ args["k"] = _integer(raw["k"], "k", 1, 10)
72
+ return args
73
+ if tool == "evidence":
74
+ args = {"source": _text(raw.get("source"), "source", 512)}
75
+ if "page" in raw and raw["page"] not in (None, ""):
76
+ args["page"] = _integer(raw["page"], "page", 1)
77
+ if "section" in raw and raw["section"] not in (None, ""):
78
+ args["section"] = _text(
79
+ raw["section"], "section", 200, required=False)
80
+ return args
81
+ if tool == "task_add":
82
+ return {"text": _text(raw.get("text"), "text", 500)}
83
+ if tool == "task_complete":
84
+ return {"id": _integer(raw.get("id"), "id", 1, 1_000_000)}
85
+ if tool == "research":
86
+ return {"query": _text(raw.get("query"), "query", 300)}
87
+ return {}
88
+
89
+
90
+ def validate_plan(steps) -> list[dict]:
91
+ """Validate and normalize a plan; fail closed on any unknown field."""
92
+ if not isinstance(steps, list) or not steps:
93
+ raise ToolPlanError("steps must be a non-empty list")
94
+ if len(steps) > MAX_STEPS:
95
+ raise ToolPlanError(f"a plan may contain at most {MAX_STEPS} steps")
96
+ plan = []
97
+ for index, raw in enumerate(steps, start=1):
98
+ if not isinstance(raw, dict):
99
+ raise ToolPlanError(f"step {index} must be an object")
100
+ unknown = set(raw) - {"tool", "args"}
101
+ if unknown:
102
+ raise ToolPlanError(
103
+ f"step {index} has unknown field(s): "
104
+ f"{', '.join(sorted(unknown))}")
105
+ tool = raw.get("tool")
106
+ if not isinstance(tool, str) or tool not in TOOL_SPECS:
107
+ raise ToolPlanError(f"step {index} has an unknown tool: {tool!r}")
108
+ plan.append({"tool": tool, "args": _validate_args(tool, raw.get("args"))})
109
+ return plan
110
+
111
+
112
+ def preview_plan(steps) -> list[dict]:
113
+ plan = validate_plan(steps)
114
+ return [{
115
+ "tool": step["tool"],
116
+ "args": step["args"],
117
+ "side_effect": bool(TOOL_SPECS[step["tool"]]["side_effect"]),
118
+ } for step in plan]
119
+
120
+
121
+ def _check_safety(session, text: str) -> None:
122
+ safety = getattr(session, "safety", None)
123
+ if safety is None:
124
+ return
125
+ verdict = safety.check(text)
126
+ if verdict is not None:
127
+ raise ToolPlanError("request blocked by the optional safety filter")
128
+
129
+
130
+ def _execute_step(session, step: dict):
131
+ tool = step["tool"]
132
+ args = step["args"]
133
+ if tool == "time":
134
+ return {
135
+ "utc": datetime.now(timezone.utc).isoformat(timespec="seconds"),
136
+ "local": datetime.now().isoformat(timespec="seconds"),
137
+ }
138
+ if tool == "calculator":
139
+ from atomzero.agent_runtime import calculate
140
+
141
+ expression = args["expression"]
142
+ return {"expression": expression, "value": calculate(expression)}
143
+ if tool == "search_docs":
144
+ _check_safety(session, args["query"])
145
+ return session._search_docs(args["query"])
146
+ if tool == "evidence":
147
+ docs = getattr(session, "docs", None)
148
+ if docs is None:
149
+ raise ToolPlanError("no document memory configured")
150
+ return docs.source_evidence(
151
+ args["source"], page=args.get("page"),
152
+ section=args.get("section", ""))
153
+ if tool == "task_list":
154
+ return {"tasks": list(session.human.data.get("tasks", []))}
155
+ if tool == "task_add":
156
+ _check_safety(session, args["text"])
157
+ tid = session.human.add_task(args["text"])
158
+ return {"id": tid, "text": args["text"]}
159
+ if tool == "task_complete":
160
+ completed = session.human.complete_task(args["id"])
161
+ return {"id": args["id"], "completed": bool(completed)}
162
+ if tool == "research":
163
+ _check_safety(session, args["query"])
164
+ return session._research(args["query"])
165
+ raise ToolPlanError(f"unknown tool: {tool}") # pragma: no cover
166
+
167
+
168
+ def execute_plan(session, steps, confirm: bool = False) -> dict:
169
+ """Execute a validated plan; side effects require ``confirm=True``.
170
+
171
+ If any step has a side effect and confirmation is missing, nothing is
172
+ executed. This avoids a half-applied plan.
173
+ """
174
+ plan = validate_plan(steps)
175
+ preview = [{
176
+ "tool": step["tool"],
177
+ "args": step["args"],
178
+ "side_effect": bool(TOOL_SPECS[step["tool"]]["side_effect"]),
179
+ } for step in plan]
180
+ side_effects = [step for step in preview if step["side_effect"]]
181
+ if side_effects and not confirm:
182
+ return {
183
+ "status": "confirmation_required",
184
+ "confirm": False,
185
+ "plan": preview,
186
+ "side_effects": side_effects,
187
+ "results": [],
188
+ }
189
+ results = []
190
+ for step in plan:
191
+ try:
192
+ value = _execute_step(session, step)
193
+ except Exception as exc:
194
+ return {
195
+ "status": "error",
196
+ "confirm": bool(confirm),
197
+ "plan": preview,
198
+ "results": results,
199
+ "error": f"{step['tool']}: {exc}",
200
+ }
201
+ results.append({"tool": step["tool"], "result": value})
202
+ return {
203
+ "status": "ok",
204
+ "confirm": bool(confirm),
205
+ "plan": preview,
206
+ "results": results,
207
+ }
@@ -0,0 +1,102 @@
1
+ """Small transformer LM and generation helpers for the chat product."""
2
+ from __future__ import annotations
3
+
4
+ import torch
5
+ import torch.nn as nn
6
+
7
+
8
+ class Block(nn.Module):
9
+ def __init__(self, dim: int, heads: int, ff: int):
10
+ super().__init__()
11
+ self.ln1 = nn.LayerNorm(dim)
12
+ self.attn = nn.MultiheadAttention(dim, heads, batch_first=True)
13
+ self.ln2 = nn.LayerNorm(dim)
14
+ self.mlp = nn.Sequential(
15
+ nn.Linear(dim, ff), nn.GELU(), nn.Linear(ff, dim))
16
+
17
+ def forward(self, x, mask):
18
+ h = self.ln1(x)
19
+ a, _ = self.attn(h, h, h, attn_mask=mask, need_weights=False)
20
+ x = x + a
21
+ x = x + self.mlp(self.ln2(x))
22
+ return x
23
+
24
+
25
+ class TransformerLM(nn.Module):
26
+ def __init__(self, vocab: int, dim: int = 512, layers: int = 5,
27
+ heads: int = 8, ff: int = 2048, max_seq: int = 512):
28
+ super().__init__()
29
+ self.vocab = vocab
30
+ self.max_seq = max_seq
31
+ self.tok = nn.Embedding(vocab, dim)
32
+ self.pos = nn.Embedding(max_seq, dim)
33
+ self.blocks = nn.ModuleList(
34
+ [Block(dim, heads, ff) for _ in range(layers)])
35
+ self.ln = nn.LayerNorm(dim)
36
+ self.head = nn.Linear(dim, vocab, bias=False)
37
+ self.head.weight = self.tok.weight
38
+ self.apply(self._init)
39
+
40
+ @staticmethod
41
+ def _init(module):
42
+ if isinstance(module, nn.Linear):
43
+ nn.init.normal_(module.weight, mean=0.0, std=0.02)
44
+ if module.bias is not None:
45
+ nn.init.zeros_(module.bias)
46
+ elif isinstance(module, nn.Embedding):
47
+ nn.init.normal_(module.weight, mean=0.0, std=0.02)
48
+
49
+ def forward(self, x):
50
+ b, t = x.shape
51
+ pos = torch.arange(t, device=x.device)
52
+ h = self.tok(x) + self.pos(pos)[None, :, :]
53
+ mask = torch.triu(
54
+ torch.full((t, t), float("-inf"), device=x.device), diagonal=1)
55
+ for block in self.blocks:
56
+ h = block(h, mask)
57
+ return self.head(self.ln(h))
58
+
59
+
60
+ @torch.no_grad()
61
+ def generate(model, tok, prompt: str, max_new: int = 80,
62
+ temperature: float = 0.8, top_k: int = 40,
63
+ repetition_penalty: float = 1.1, banned_ids=(),
64
+ device=None) -> tuple[str, str]:
65
+ dev = device or next(model.parameters()).device
66
+ model.eval()
67
+ end_id = tok.token_to_id("<|end|>")
68
+ user_id = tok.token_to_id("<|user|>")
69
+ assistant_id = tok.token_to_id("<|assistant|>")
70
+ prompt_ids = tok.encode(prompt).ids
71
+ ids = list(prompt_ids)
72
+ seen = set(ids)
73
+ stop = "max"
74
+ for _ in range(max_new):
75
+ x = torch.tensor([ids[-model.max_seq:]], device=dev)
76
+ logits = model(x)[0, -1].float().clone()
77
+ if banned_ids:
78
+ logits[list(banned_ids)] = float("-inf")
79
+ if repetition_penalty != 1.0:
80
+ for tid in seen:
81
+ if logits[tid] > 0:
82
+ logits[tid] /= repetition_penalty
83
+ else:
84
+ logits[tid] *= repetition_penalty
85
+ logits = logits / max(temperature, 1e-5)
86
+ if top_k > 0:
87
+ k = min(top_k, logits.numel())
88
+ vals, idx = torch.topk(logits, k)
89
+ logits = torch.full_like(logits, float("-inf"))
90
+ logits[idx] = vals
91
+ probs = torch.softmax(logits, dim=-1)
92
+ nxt = int(torch.multinomial(probs, 1).item())
93
+ if nxt == end_id:
94
+ stop = "end"
95
+ break
96
+ if nxt in (user_id, assistant_id):
97
+ stop = "tag"
98
+ break
99
+ ids.append(nxt)
100
+ seen.add(nxt)
101
+ text = tok.decode(ids[len(prompt_ids):], skip_special_tokens=False)
102
+ return text.strip(), stop
@@ -0,0 +1,165 @@
1
+ """Safe, source-cited web research for the local document index.
2
+
3
+ No autonomous crawler. Wikipedia is the default research source; arbitrary URL
4
+ fetching is opt-in and blocked for private/loopback hosts by default.
5
+ """
6
+ from __future__ import annotations
7
+
8
+ import ipaddress
9
+ import json
10
+ import re
11
+ import socket
12
+ import urllib.parse
13
+ import urllib.request
14
+ from html.parser import HTMLParser
15
+
16
+ USER_AGENT = "streamft-local-assistant/0.1 (local research demo)"
17
+ MAX_BYTES = 2_000_000
18
+
19
+
20
+ class _Text(HTMLParser):
21
+ def __init__(self):
22
+ super().__init__()
23
+ self.title = ""
24
+ self.parts = []
25
+ self._skip = 0
26
+ self._in_title = False
27
+
28
+ def handle_starttag(self, tag, attrs):
29
+ tag = tag.lower()
30
+ if tag in ("script", "style", "noscript", "svg"):
31
+ self._skip += 1
32
+ if tag == "title":
33
+ self._in_title = True
34
+
35
+ def handle_endtag(self, tag):
36
+ tag = tag.lower()
37
+ if tag in ("script", "style", "noscript", "svg") and self._skip:
38
+ self._skip -= 1
39
+ if tag == "title":
40
+ self._in_title = False
41
+
42
+ def handle_data(self, data):
43
+ if self._in_title:
44
+ self.title += data
45
+ elif not self._skip:
46
+ text = re.sub(r"\s+", " ", data).strip()
47
+ if text:
48
+ self.parts.append(text)
49
+
50
+
51
+ def html_to_text(html: str) -> tuple[str, str]:
52
+ parser = _Text()
53
+ parser.feed(html)
54
+ text = "\n\n".join(parser.parts)
55
+ return parser.title.strip(), text
56
+
57
+
58
+ def _check_public(url: str, allow_private: bool) -> None:
59
+ parsed = urllib.parse.urlparse(url)
60
+ if parsed.scheme not in ("http", "https"):
61
+ raise ValueError("only http/https URLs are allowed")
62
+ if not parsed.hostname:
63
+ raise ValueError("URL has no host")
64
+ if allow_private:
65
+ return
66
+ try:
67
+ infos = socket.getaddrinfo(parsed.hostname, None)
68
+ except OSError as exc:
69
+ raise ValueError(f"cannot resolve host: {exc}") from exc
70
+ for info in infos:
71
+ ip = ipaddress.ip_address(info[4][0])
72
+ if (ip.is_private or ip.is_loopback or ip.is_link_local
73
+ or ip.is_reserved):
74
+ raise ValueError("private/loopback hosts are blocked")
75
+
76
+
77
+ def fetch_url(url: str, allow_private: bool = False,
78
+ timeout: int = 15) -> tuple[str, str, str]:
79
+ _check_public(url, allow_private)
80
+ req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
81
+ with urllib.request.urlopen(req, timeout=timeout) as response:
82
+ ctype = response.headers.get("Content-Type", "")
83
+ raw = response.read(MAX_BYTES + 1)
84
+ final = response.geturl()
85
+ if len(raw) > MAX_BYTES:
86
+ raise ValueError("page is too large")
87
+ if "html" in ctype:
88
+ text = raw.decode("utf-8", errors="replace")
89
+ else:
90
+ text = raw.decode("utf-8", errors="replace")
91
+ return final, text, ctype
92
+
93
+
94
+ def index_url(memory, url: str, title: str = "",
95
+ allow_private: bool = False) -> dict:
96
+ final, body, ctype = fetch_url(url, allow_private=allow_private)
97
+ if "html" in ctype:
98
+ parsed_title, text = html_to_text(body)
99
+ else:
100
+ parsed_title, text = "", body
101
+ text = text.strip()
102
+ if len(text) < 20:
103
+ raise ValueError("page has no usable text")
104
+ title = title or parsed_title or final
105
+ chunks = memory.add_text(final, text, title=title)
106
+ return {"source": final, "title": title, "chunks": chunks,
107
+ "preview": text[:400]}
108
+
109
+
110
+ def wikipedia_research(memory, query: str, lang: str = "en",
111
+ limit: int = 2, timeout: int = 20,
112
+ intro_only: bool = True) -> list[dict]:
113
+ api = f"https://{lang}.wikipedia.org/w/api.php"
114
+ params = {
115
+ "action": "query", "list": "search", "srsearch": query,
116
+ "format": "json", "srlimit": max(3, min(10, limit * 2)),
117
+ }
118
+ req = urllib.request.Request(
119
+ api + "?" + urllib.parse.urlencode(params),
120
+ headers={"User-Agent": USER_AGENT})
121
+ with urllib.request.urlopen(req, timeout=timeout) as response:
122
+ data = json.loads(response.read().decode("utf-8"))
123
+ results = [result for result in
124
+ data.get("query", {}).get("search", [])
125
+ if result.get("pageid")]
126
+ wanted = query.strip().lower()
127
+
128
+ def rank(item):
129
+ index, result = item
130
+ title = str(result.get("title", "")).strip().lower()
131
+ if title == wanted:
132
+ return (0, index)
133
+ if title.startswith(wanted + " ("):
134
+ return (1, index)
135
+ if title.startswith(wanted):
136
+ return (2, index)
137
+ return (3, index)
138
+
139
+ results = [result for _, result in
140
+ sorted(enumerate(results), key=rank)][:max(1, min(limit, 5))]
141
+ pages = []
142
+ for result in results:
143
+ pageid = result.get("pageid")
144
+ title = result.get("title", "")
145
+ if not pageid:
146
+ continue
147
+ extract_params = {
148
+ "action": "query", "prop": "extracts", "explaintext": 1,
149
+ "exintro": 1 if intro_only else 0,
150
+ "pageids": pageid, "format": "json",
151
+ }
152
+ req = urllib.request.Request(
153
+ api + "?" + urllib.parse.urlencode(extract_params),
154
+ headers={"User-Agent": USER_AGENT})
155
+ with urllib.request.urlopen(req, timeout=timeout) as response:
156
+ extract = json.loads(response.read().decode("utf-8"))
157
+ page = extract.get("query", {}).get("pages", {}).get(str(pageid), {})
158
+ text = (page.get("extract") or "").strip()
159
+ if len(text) < 40:
160
+ continue
161
+ source = f"https://{lang}.wikipedia.org/?curid={pageid}"
162
+ chunks = memory.add_text(source, text, title=title)
163
+ pages.append({"source": source, "title": title, "chunks": chunks,
164
+ "preview": text[:400], "context": text[:4000]})
165
+ return pages