atomzero 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- atomzero/__init__.py +8 -0
- atomzero/agent_runtime.py +225 -0
- atomzero/chat_human.py +486 -0
- atomzero/chat_server.py +2494 -0
- atomzero/curiosity.py +66 -0
- atomzero/doc_ingest.py +312 -0
- atomzero/doc_memory.py +1112 -0
- atomzero/feedback.py +188 -0
- atomzero/knowledge_gaps.py +157 -0
- atomzero/lora.py +140 -0
- atomzero/ml_safety.py +165 -0
- atomzero/neural_embedder.py +108 -0
- atomzero/research_queue.py +330 -0
- atomzero/safety.py +226 -0
- atomzero/teacher_notes.py +78 -0
- atomzero/teacher_provider.py +104 -0
- atomzero/tool_planner.py +207 -0
- atomzero/transformer.py +102 -0
- atomzero/web_research.py +165 -0
- atomzero-0.1.0.dist-info/METADATA +96 -0
- atomzero-0.1.0.dist-info/RECORD +26 -0
- atomzero-0.1.0.dist-info/WHEEL +5 -0
- atomzero-0.1.0.dist-info/entry_points.txt +2 -0
- atomzero-0.1.0.dist-info/licenses/LICENSE +202 -0
- atomzero-0.1.0.dist-info/licenses/NOTICE +24 -0
- atomzero-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
"""Store manually supplied teacher answers as unverified RAG data.
|
|
2
|
+
|
|
3
|
+
This module is shared by the HTTP server and the reviewed research queue so both
|
|
4
|
+
paths use the same provenance, limits and `verified=false` marking.
|
|
5
|
+
"""
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import hashlib
|
|
9
|
+
import re
|
|
10
|
+
import urllib.parse
|
|
11
|
+
|
|
12
|
+
MAX_TEACHER_BYTES = 200_000
|
|
13
|
+
MAX_TEACHER_TOPIC = 300
|
|
14
|
+
MAX_TEACHER_LABEL = 100
|
|
15
|
+
MAX_SOURCE_URL = 2048
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class TeacherNoteTooLarge(ValueError):
|
|
19
|
+
pass
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def add_teacher_note(docs, teacher, model, topic, answer,
|
|
23
|
+
source_url: str = "") -> dict:
|
|
24
|
+
"""Validate and store a teacher answer with derived/unverified metadata."""
|
|
25
|
+
if docs is None:
|
|
26
|
+
raise ValueError("no document memory configured")
|
|
27
|
+
teacher = str(teacher or "manual").strip() or "manual"
|
|
28
|
+
model = str(model or "").strip()
|
|
29
|
+
topic = str(topic or "").strip()
|
|
30
|
+
answer = str(answer or "").strip()
|
|
31
|
+
source_url = str(source_url or "").strip()
|
|
32
|
+
for label, value, limit in (
|
|
33
|
+
("teacher", teacher, MAX_TEACHER_LABEL),
|
|
34
|
+
("model", model, MAX_TEACHER_LABEL),
|
|
35
|
+
("topic", topic, MAX_TEACHER_TOPIC)):
|
|
36
|
+
if len(value) > limit:
|
|
37
|
+
raise ValueError(f"{label} is too long (max {limit} chars)")
|
|
38
|
+
if any(ord(ch) < 32 for ch in value):
|
|
39
|
+
raise ValueError(f"{label} contains control characters")
|
|
40
|
+
if not answer:
|
|
41
|
+
raise ValueError("answer is required")
|
|
42
|
+
if source_url:
|
|
43
|
+
if len(source_url) > MAX_SOURCE_URL:
|
|
44
|
+
raise ValueError(
|
|
45
|
+
f"source_url is too long (max {MAX_SOURCE_URL} chars)")
|
|
46
|
+
parsed = urllib.parse.urlparse(source_url)
|
|
47
|
+
if parsed.scheme not in ("http", "https") or not parsed.hostname:
|
|
48
|
+
raise ValueError("source_url must be an http(s) URL")
|
|
49
|
+
text = f"Topic: {topic}\n\n{answer}" if topic else answer
|
|
50
|
+
raw = text.encode("utf-8")
|
|
51
|
+
if len(raw) > MAX_TEACHER_BYTES:
|
|
52
|
+
raise TeacherNoteTooLarge(
|
|
53
|
+
f"teacher note is too large (max {MAX_TEACHER_BYTES} bytes)")
|
|
54
|
+
safe = re.sub(r"[^A-Za-z0-9_.-]+", "-", teacher).strip("-")[:48]
|
|
55
|
+
digest = hashlib.sha256(raw).hexdigest()
|
|
56
|
+
source = f"teacher://{safe or 'manual'}/{digest[:16]}"
|
|
57
|
+
title = (f"Teacher note: {topic}" if topic
|
|
58
|
+
else f"Teacher note ({teacher})")
|
|
59
|
+
chunks = docs.add_text(source, text, title=title, meta={
|
|
60
|
+
"kind": "teacher",
|
|
61
|
+
"source_url": source_url,
|
|
62
|
+
"teacher": teacher,
|
|
63
|
+
"model": model,
|
|
64
|
+
"verified": False,
|
|
65
|
+
})
|
|
66
|
+
return {
|
|
67
|
+
"source": source,
|
|
68
|
+
"title": title,
|
|
69
|
+
"chunks": chunks,
|
|
70
|
+
"bytes": len(raw),
|
|
71
|
+
"sha256": digest,
|
|
72
|
+
"kind": "teacher",
|
|
73
|
+
"verified": False,
|
|
74
|
+
"source_url": source_url,
|
|
75
|
+
"teacher": teacher,
|
|
76
|
+
"model": model,
|
|
77
|
+
"note": "teacher-derived and unverified",
|
|
78
|
+
}
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
"""Optional OpenAI-compatible teacher provider.
|
|
2
|
+
|
|
3
|
+
Point ``base_url`` at a local Ollama OpenAI-compatible endpoint or another
|
|
4
|
+
compatible server. The provider is only called by an explicit request; it is
|
|
5
|
+
never used automatically and its output is stored as unverified teacher data.
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import json
|
|
10
|
+
import urllib.error
|
|
11
|
+
import urllib.parse
|
|
12
|
+
import urllib.request
|
|
13
|
+
|
|
14
|
+
USER_AGENT = "streamft-local-assistant/0.1"
|
|
15
|
+
MAX_RESPONSE_BYTES = 2_000_000
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class TeacherProviderError(RuntimeError):
|
|
19
|
+
pass
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class TeacherProvider:
|
|
23
|
+
def __init__(self, base_url: str, model: str, api_key: str = "",
|
|
24
|
+
timeout: int = 60, label: str = "openai-compatible"):
|
|
25
|
+
base_url = str(base_url or "").strip()
|
|
26
|
+
model = str(model or "").strip()
|
|
27
|
+
if not base_url or not model:
|
|
28
|
+
raise TeacherProviderError(
|
|
29
|
+
"teacher base URL and model are required")
|
|
30
|
+
parsed = urllib.parse.urlparse(base_url)
|
|
31
|
+
if parsed.scheme not in ("http", "https") or not parsed.hostname:
|
|
32
|
+
raise TeacherProviderError("teacher URL must be an http(s) URL")
|
|
33
|
+
self.base_url = base_url.rstrip("/")
|
|
34
|
+
self.model = model
|
|
35
|
+
self.api_key = str(api_key or "")
|
|
36
|
+
try:
|
|
37
|
+
self.timeout = max(1, min(int(timeout), 300))
|
|
38
|
+
except (TypeError, ValueError) as exc:
|
|
39
|
+
raise TeacherProviderError(
|
|
40
|
+
"teacher timeout must be an integer") from exc
|
|
41
|
+
self.label = str(label or "openai-compatible")[:100]
|
|
42
|
+
self.host = parsed.netloc
|
|
43
|
+
self.endpoint = (
|
|
44
|
+
self.base_url if self.base_url.endswith("/chat/completions")
|
|
45
|
+
else self.base_url + "/chat/completions")
|
|
46
|
+
|
|
47
|
+
def ask(self, prompt: str, system: str = "",
|
|
48
|
+
max_tokens: int = 512) -> str:
|
|
49
|
+
prompt = str(prompt or "").strip()
|
|
50
|
+
if not prompt:
|
|
51
|
+
raise TeacherProviderError("prompt is required")
|
|
52
|
+
try:
|
|
53
|
+
max_tokens = max(1, min(int(max_tokens), 2048))
|
|
54
|
+
except (TypeError, ValueError) as exc:
|
|
55
|
+
raise TeacherProviderError("max_tokens must be an integer") from exc
|
|
56
|
+
messages = []
|
|
57
|
+
if system:
|
|
58
|
+
messages.append({"role": "system", "content": str(system)})
|
|
59
|
+
messages.append({"role": "user", "content": prompt})
|
|
60
|
+
body = {
|
|
61
|
+
"model": self.model,
|
|
62
|
+
"messages": messages,
|
|
63
|
+
"temperature": 0.2,
|
|
64
|
+
"max_tokens": max_tokens,
|
|
65
|
+
}
|
|
66
|
+
headers = {
|
|
67
|
+
"Content-Type": "application/json",
|
|
68
|
+
"User-Agent": USER_AGENT,
|
|
69
|
+
}
|
|
70
|
+
if self.api_key:
|
|
71
|
+
headers["Authorization"] = f"Bearer {self.api_key}"
|
|
72
|
+
request = urllib.request.Request(
|
|
73
|
+
self.endpoint, data=json.dumps(body).encode("utf-8"),
|
|
74
|
+
headers=headers, method="POST")
|
|
75
|
+
try:
|
|
76
|
+
with urllib.request.urlopen(
|
|
77
|
+
request, timeout=self.timeout) as response:
|
|
78
|
+
raw = response.read(MAX_RESPONSE_BYTES + 1)
|
|
79
|
+
except urllib.error.HTTPError as exc:
|
|
80
|
+
try:
|
|
81
|
+
detail = exc.read(500).decode("utf-8", errors="replace")
|
|
82
|
+
except Exception:
|
|
83
|
+
detail = ""
|
|
84
|
+
raise TeacherProviderError(
|
|
85
|
+
f"teacher provider HTTP {exc.code}: {detail}") from exc
|
|
86
|
+
except Exception as exc:
|
|
87
|
+
raise TeacherProviderError(
|
|
88
|
+
f"teacher provider request failed: {exc}") from exc
|
|
89
|
+
if len(raw) > MAX_RESPONSE_BYTES:
|
|
90
|
+
raise TeacherProviderError("teacher provider response is too large")
|
|
91
|
+
try:
|
|
92
|
+
data = json.loads(raw.decode("utf-8"))
|
|
93
|
+
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
|
94
|
+
raise TeacherProviderError(
|
|
95
|
+
"teacher provider returned invalid JSON") from exc
|
|
96
|
+
try:
|
|
97
|
+
content = data["choices"][0]["message"]["content"]
|
|
98
|
+
except (KeyError, IndexError, TypeError) as exc:
|
|
99
|
+
raise TeacherProviderError(
|
|
100
|
+
"teacher provider response has no message content") from exc
|
|
101
|
+
content = str(content or "").strip()
|
|
102
|
+
if not content:
|
|
103
|
+
raise TeacherProviderError("teacher provider returned an empty answer")
|
|
104
|
+
return content
|
atomzero/tool_planner.py
ADDED
|
@@ -0,0 +1,207 @@
|
|
|
1
|
+
"""Constrained tool-plan validation and execution.
|
|
2
|
+
|
|
3
|
+
This is an allowlisted executor, not free-form model planning. A caller (the
|
|
4
|
+
UI, a test, or a future provider-assisted planner) submits a small JSON plan;
|
|
5
|
+
this module rejects unknown tools/arguments, caps the plan at three steps and
|
|
6
|
+
requires explicit confirmation before any side effect.
|
|
7
|
+
|
|
8
|
+
The 49.2M core does not choose these steps. That boundary is deliberate: a
|
|
9
|
+
small model can propose a plan later, but it cannot bypass this validator.
|
|
10
|
+
"""
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
from datetime import datetime, timezone
|
|
14
|
+
|
|
15
|
+
MAX_STEPS = 3
|
|
16
|
+
|
|
17
|
+
TOOL_SPECS = {
|
|
18
|
+
"time": {"side_effect": False, "args": ()},
|
|
19
|
+
"calculator": {"side_effect": False, "args": ("expression",)},
|
|
20
|
+
"search_docs": {"side_effect": False, "args": ("query", "k")},
|
|
21
|
+
"evidence": {"side_effect": False, "args": ("source", "page", "section")},
|
|
22
|
+
"task_list": {"side_effect": False, "args": ()},
|
|
23
|
+
"task_add": {"side_effect": True, "args": ("text",)},
|
|
24
|
+
"task_complete": {"side_effect": True, "args": ("id",)},
|
|
25
|
+
"research": {"side_effect": True, "args": ("query",)},
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class ToolPlanError(ValueError):
|
|
30
|
+
"""Raised for an invalid plan or an unsafe tool invocation."""
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _text(value, name: str, maximum: int, required: bool = True) -> str:
|
|
34
|
+
if not isinstance(value, str):
|
|
35
|
+
raise ToolPlanError(f"{name} must be a string")
|
|
36
|
+
value = value.strip()
|
|
37
|
+
if required and not value:
|
|
38
|
+
raise ToolPlanError(f"{name} is required")
|
|
39
|
+
if len(value) > maximum:
|
|
40
|
+
raise ToolPlanError(f"{name} is too long (max {maximum} chars)")
|
|
41
|
+
if any(ord(ch) < 32 and ch not in "\n\t" for ch in value):
|
|
42
|
+
raise ToolPlanError(f"{name} contains control characters")
|
|
43
|
+
return value
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _integer(value, name: str, minimum: int = 1,
|
|
47
|
+
maximum: int = 1_000_000) -> int:
|
|
48
|
+
if isinstance(value, bool) or not isinstance(value, int):
|
|
49
|
+
raise ToolPlanError(f"{name} must be an integer")
|
|
50
|
+
if value < minimum or value > maximum:
|
|
51
|
+
raise ToolPlanError(
|
|
52
|
+
f"{name} must be between {minimum} and {maximum}")
|
|
53
|
+
return value
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _validate_args(tool: str, raw) -> dict:
|
|
57
|
+
if raw is None:
|
|
58
|
+
raw = {}
|
|
59
|
+
if not isinstance(raw, dict):
|
|
60
|
+
raise ToolPlanError("args must be an object")
|
|
61
|
+
allowed = set(TOOL_SPECS[tool]["args"])
|
|
62
|
+
unknown = set(raw) - allowed
|
|
63
|
+
if unknown:
|
|
64
|
+
raise ToolPlanError(
|
|
65
|
+
f"unknown argument(s) for {tool}: {', '.join(sorted(unknown))}")
|
|
66
|
+
if tool == "calculator":
|
|
67
|
+
return {"expression": _text(raw.get("expression"), "expression", 80)}
|
|
68
|
+
if tool == "search_docs":
|
|
69
|
+
args = {"query": _text(raw.get("query"), "query", 500)}
|
|
70
|
+
if "k" in raw:
|
|
71
|
+
args["k"] = _integer(raw["k"], "k", 1, 10)
|
|
72
|
+
return args
|
|
73
|
+
if tool == "evidence":
|
|
74
|
+
args = {"source": _text(raw.get("source"), "source", 512)}
|
|
75
|
+
if "page" in raw and raw["page"] not in (None, ""):
|
|
76
|
+
args["page"] = _integer(raw["page"], "page", 1)
|
|
77
|
+
if "section" in raw and raw["section"] not in (None, ""):
|
|
78
|
+
args["section"] = _text(
|
|
79
|
+
raw["section"], "section", 200, required=False)
|
|
80
|
+
return args
|
|
81
|
+
if tool == "task_add":
|
|
82
|
+
return {"text": _text(raw.get("text"), "text", 500)}
|
|
83
|
+
if tool == "task_complete":
|
|
84
|
+
return {"id": _integer(raw.get("id"), "id", 1, 1_000_000)}
|
|
85
|
+
if tool == "research":
|
|
86
|
+
return {"query": _text(raw.get("query"), "query", 300)}
|
|
87
|
+
return {}
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def validate_plan(steps) -> list[dict]:
|
|
91
|
+
"""Validate and normalize a plan; fail closed on any unknown field."""
|
|
92
|
+
if not isinstance(steps, list) or not steps:
|
|
93
|
+
raise ToolPlanError("steps must be a non-empty list")
|
|
94
|
+
if len(steps) > MAX_STEPS:
|
|
95
|
+
raise ToolPlanError(f"a plan may contain at most {MAX_STEPS} steps")
|
|
96
|
+
plan = []
|
|
97
|
+
for index, raw in enumerate(steps, start=1):
|
|
98
|
+
if not isinstance(raw, dict):
|
|
99
|
+
raise ToolPlanError(f"step {index} must be an object")
|
|
100
|
+
unknown = set(raw) - {"tool", "args"}
|
|
101
|
+
if unknown:
|
|
102
|
+
raise ToolPlanError(
|
|
103
|
+
f"step {index} has unknown field(s): "
|
|
104
|
+
f"{', '.join(sorted(unknown))}")
|
|
105
|
+
tool = raw.get("tool")
|
|
106
|
+
if not isinstance(tool, str) or tool not in TOOL_SPECS:
|
|
107
|
+
raise ToolPlanError(f"step {index} has an unknown tool: {tool!r}")
|
|
108
|
+
plan.append({"tool": tool, "args": _validate_args(tool, raw.get("args"))})
|
|
109
|
+
return plan
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def preview_plan(steps) -> list[dict]:
|
|
113
|
+
plan = validate_plan(steps)
|
|
114
|
+
return [{
|
|
115
|
+
"tool": step["tool"],
|
|
116
|
+
"args": step["args"],
|
|
117
|
+
"side_effect": bool(TOOL_SPECS[step["tool"]]["side_effect"]),
|
|
118
|
+
} for step in plan]
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def _check_safety(session, text: str) -> None:
|
|
122
|
+
safety = getattr(session, "safety", None)
|
|
123
|
+
if safety is None:
|
|
124
|
+
return
|
|
125
|
+
verdict = safety.check(text)
|
|
126
|
+
if verdict is not None:
|
|
127
|
+
raise ToolPlanError("request blocked by the optional safety filter")
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def _execute_step(session, step: dict):
|
|
131
|
+
tool = step["tool"]
|
|
132
|
+
args = step["args"]
|
|
133
|
+
if tool == "time":
|
|
134
|
+
return {
|
|
135
|
+
"utc": datetime.now(timezone.utc).isoformat(timespec="seconds"),
|
|
136
|
+
"local": datetime.now().isoformat(timespec="seconds"),
|
|
137
|
+
}
|
|
138
|
+
if tool == "calculator":
|
|
139
|
+
from atomzero.agent_runtime import calculate
|
|
140
|
+
|
|
141
|
+
expression = args["expression"]
|
|
142
|
+
return {"expression": expression, "value": calculate(expression)}
|
|
143
|
+
if tool == "search_docs":
|
|
144
|
+
_check_safety(session, args["query"])
|
|
145
|
+
return session._search_docs(args["query"])
|
|
146
|
+
if tool == "evidence":
|
|
147
|
+
docs = getattr(session, "docs", None)
|
|
148
|
+
if docs is None:
|
|
149
|
+
raise ToolPlanError("no document memory configured")
|
|
150
|
+
return docs.source_evidence(
|
|
151
|
+
args["source"], page=args.get("page"),
|
|
152
|
+
section=args.get("section", ""))
|
|
153
|
+
if tool == "task_list":
|
|
154
|
+
return {"tasks": list(session.human.data.get("tasks", []))}
|
|
155
|
+
if tool == "task_add":
|
|
156
|
+
_check_safety(session, args["text"])
|
|
157
|
+
tid = session.human.add_task(args["text"])
|
|
158
|
+
return {"id": tid, "text": args["text"]}
|
|
159
|
+
if tool == "task_complete":
|
|
160
|
+
completed = session.human.complete_task(args["id"])
|
|
161
|
+
return {"id": args["id"], "completed": bool(completed)}
|
|
162
|
+
if tool == "research":
|
|
163
|
+
_check_safety(session, args["query"])
|
|
164
|
+
return session._research(args["query"])
|
|
165
|
+
raise ToolPlanError(f"unknown tool: {tool}") # pragma: no cover
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def execute_plan(session, steps, confirm: bool = False) -> dict:
|
|
169
|
+
"""Execute a validated plan; side effects require ``confirm=True``.
|
|
170
|
+
|
|
171
|
+
If any step has a side effect and confirmation is missing, nothing is
|
|
172
|
+
executed. This avoids a half-applied plan.
|
|
173
|
+
"""
|
|
174
|
+
plan = validate_plan(steps)
|
|
175
|
+
preview = [{
|
|
176
|
+
"tool": step["tool"],
|
|
177
|
+
"args": step["args"],
|
|
178
|
+
"side_effect": bool(TOOL_SPECS[step["tool"]]["side_effect"]),
|
|
179
|
+
} for step in plan]
|
|
180
|
+
side_effects = [step for step in preview if step["side_effect"]]
|
|
181
|
+
if side_effects and not confirm:
|
|
182
|
+
return {
|
|
183
|
+
"status": "confirmation_required",
|
|
184
|
+
"confirm": False,
|
|
185
|
+
"plan": preview,
|
|
186
|
+
"side_effects": side_effects,
|
|
187
|
+
"results": [],
|
|
188
|
+
}
|
|
189
|
+
results = []
|
|
190
|
+
for step in plan:
|
|
191
|
+
try:
|
|
192
|
+
value = _execute_step(session, step)
|
|
193
|
+
except Exception as exc:
|
|
194
|
+
return {
|
|
195
|
+
"status": "error",
|
|
196
|
+
"confirm": bool(confirm),
|
|
197
|
+
"plan": preview,
|
|
198
|
+
"results": results,
|
|
199
|
+
"error": f"{step['tool']}: {exc}",
|
|
200
|
+
}
|
|
201
|
+
results.append({"tool": step["tool"], "result": value})
|
|
202
|
+
return {
|
|
203
|
+
"status": "ok",
|
|
204
|
+
"confirm": bool(confirm),
|
|
205
|
+
"plan": preview,
|
|
206
|
+
"results": results,
|
|
207
|
+
}
|
atomzero/transformer.py
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
"""Small transformer LM and generation helpers for the chat product."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import torch
|
|
5
|
+
import torch.nn as nn
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class Block(nn.Module):
|
|
9
|
+
def __init__(self, dim: int, heads: int, ff: int):
|
|
10
|
+
super().__init__()
|
|
11
|
+
self.ln1 = nn.LayerNorm(dim)
|
|
12
|
+
self.attn = nn.MultiheadAttention(dim, heads, batch_first=True)
|
|
13
|
+
self.ln2 = nn.LayerNorm(dim)
|
|
14
|
+
self.mlp = nn.Sequential(
|
|
15
|
+
nn.Linear(dim, ff), nn.GELU(), nn.Linear(ff, dim))
|
|
16
|
+
|
|
17
|
+
def forward(self, x, mask):
|
|
18
|
+
h = self.ln1(x)
|
|
19
|
+
a, _ = self.attn(h, h, h, attn_mask=mask, need_weights=False)
|
|
20
|
+
x = x + a
|
|
21
|
+
x = x + self.mlp(self.ln2(x))
|
|
22
|
+
return x
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class TransformerLM(nn.Module):
|
|
26
|
+
def __init__(self, vocab: int, dim: int = 512, layers: int = 5,
|
|
27
|
+
heads: int = 8, ff: int = 2048, max_seq: int = 512):
|
|
28
|
+
super().__init__()
|
|
29
|
+
self.vocab = vocab
|
|
30
|
+
self.max_seq = max_seq
|
|
31
|
+
self.tok = nn.Embedding(vocab, dim)
|
|
32
|
+
self.pos = nn.Embedding(max_seq, dim)
|
|
33
|
+
self.blocks = nn.ModuleList(
|
|
34
|
+
[Block(dim, heads, ff) for _ in range(layers)])
|
|
35
|
+
self.ln = nn.LayerNorm(dim)
|
|
36
|
+
self.head = nn.Linear(dim, vocab, bias=False)
|
|
37
|
+
self.head.weight = self.tok.weight
|
|
38
|
+
self.apply(self._init)
|
|
39
|
+
|
|
40
|
+
@staticmethod
|
|
41
|
+
def _init(module):
|
|
42
|
+
if isinstance(module, nn.Linear):
|
|
43
|
+
nn.init.normal_(module.weight, mean=0.0, std=0.02)
|
|
44
|
+
if module.bias is not None:
|
|
45
|
+
nn.init.zeros_(module.bias)
|
|
46
|
+
elif isinstance(module, nn.Embedding):
|
|
47
|
+
nn.init.normal_(module.weight, mean=0.0, std=0.02)
|
|
48
|
+
|
|
49
|
+
def forward(self, x):
|
|
50
|
+
b, t = x.shape
|
|
51
|
+
pos = torch.arange(t, device=x.device)
|
|
52
|
+
h = self.tok(x) + self.pos(pos)[None, :, :]
|
|
53
|
+
mask = torch.triu(
|
|
54
|
+
torch.full((t, t), float("-inf"), device=x.device), diagonal=1)
|
|
55
|
+
for block in self.blocks:
|
|
56
|
+
h = block(h, mask)
|
|
57
|
+
return self.head(self.ln(h))
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
@torch.no_grad()
|
|
61
|
+
def generate(model, tok, prompt: str, max_new: int = 80,
|
|
62
|
+
temperature: float = 0.8, top_k: int = 40,
|
|
63
|
+
repetition_penalty: float = 1.1, banned_ids=(),
|
|
64
|
+
device=None) -> tuple[str, str]:
|
|
65
|
+
dev = device or next(model.parameters()).device
|
|
66
|
+
model.eval()
|
|
67
|
+
end_id = tok.token_to_id("<|end|>")
|
|
68
|
+
user_id = tok.token_to_id("<|user|>")
|
|
69
|
+
assistant_id = tok.token_to_id("<|assistant|>")
|
|
70
|
+
prompt_ids = tok.encode(prompt).ids
|
|
71
|
+
ids = list(prompt_ids)
|
|
72
|
+
seen = set(ids)
|
|
73
|
+
stop = "max"
|
|
74
|
+
for _ in range(max_new):
|
|
75
|
+
x = torch.tensor([ids[-model.max_seq:]], device=dev)
|
|
76
|
+
logits = model(x)[0, -1].float().clone()
|
|
77
|
+
if banned_ids:
|
|
78
|
+
logits[list(banned_ids)] = float("-inf")
|
|
79
|
+
if repetition_penalty != 1.0:
|
|
80
|
+
for tid in seen:
|
|
81
|
+
if logits[tid] > 0:
|
|
82
|
+
logits[tid] /= repetition_penalty
|
|
83
|
+
else:
|
|
84
|
+
logits[tid] *= repetition_penalty
|
|
85
|
+
logits = logits / max(temperature, 1e-5)
|
|
86
|
+
if top_k > 0:
|
|
87
|
+
k = min(top_k, logits.numel())
|
|
88
|
+
vals, idx = torch.topk(logits, k)
|
|
89
|
+
logits = torch.full_like(logits, float("-inf"))
|
|
90
|
+
logits[idx] = vals
|
|
91
|
+
probs = torch.softmax(logits, dim=-1)
|
|
92
|
+
nxt = int(torch.multinomial(probs, 1).item())
|
|
93
|
+
if nxt == end_id:
|
|
94
|
+
stop = "end"
|
|
95
|
+
break
|
|
96
|
+
if nxt in (user_id, assistant_id):
|
|
97
|
+
stop = "tag"
|
|
98
|
+
break
|
|
99
|
+
ids.append(nxt)
|
|
100
|
+
seen.add(nxt)
|
|
101
|
+
text = tok.decode(ids[len(prompt_ids):], skip_special_tokens=False)
|
|
102
|
+
return text.strip(), stop
|
atomzero/web_research.py
ADDED
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
"""Safe, source-cited web research for the local document index.
|
|
2
|
+
|
|
3
|
+
No autonomous crawler. Wikipedia is the default research source; arbitrary URL
|
|
4
|
+
fetching is opt-in and blocked for private/loopback hosts by default.
|
|
5
|
+
"""
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import ipaddress
|
|
9
|
+
import json
|
|
10
|
+
import re
|
|
11
|
+
import socket
|
|
12
|
+
import urllib.parse
|
|
13
|
+
import urllib.request
|
|
14
|
+
from html.parser import HTMLParser
|
|
15
|
+
|
|
16
|
+
USER_AGENT = "streamft-local-assistant/0.1 (local research demo)"
|
|
17
|
+
MAX_BYTES = 2_000_000
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class _Text(HTMLParser):
|
|
21
|
+
def __init__(self):
|
|
22
|
+
super().__init__()
|
|
23
|
+
self.title = ""
|
|
24
|
+
self.parts = []
|
|
25
|
+
self._skip = 0
|
|
26
|
+
self._in_title = False
|
|
27
|
+
|
|
28
|
+
def handle_starttag(self, tag, attrs):
|
|
29
|
+
tag = tag.lower()
|
|
30
|
+
if tag in ("script", "style", "noscript", "svg"):
|
|
31
|
+
self._skip += 1
|
|
32
|
+
if tag == "title":
|
|
33
|
+
self._in_title = True
|
|
34
|
+
|
|
35
|
+
def handle_endtag(self, tag):
|
|
36
|
+
tag = tag.lower()
|
|
37
|
+
if tag in ("script", "style", "noscript", "svg") and self._skip:
|
|
38
|
+
self._skip -= 1
|
|
39
|
+
if tag == "title":
|
|
40
|
+
self._in_title = False
|
|
41
|
+
|
|
42
|
+
def handle_data(self, data):
|
|
43
|
+
if self._in_title:
|
|
44
|
+
self.title += data
|
|
45
|
+
elif not self._skip:
|
|
46
|
+
text = re.sub(r"\s+", " ", data).strip()
|
|
47
|
+
if text:
|
|
48
|
+
self.parts.append(text)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def html_to_text(html: str) -> tuple[str, str]:
|
|
52
|
+
parser = _Text()
|
|
53
|
+
parser.feed(html)
|
|
54
|
+
text = "\n\n".join(parser.parts)
|
|
55
|
+
return parser.title.strip(), text
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _check_public(url: str, allow_private: bool) -> None:
|
|
59
|
+
parsed = urllib.parse.urlparse(url)
|
|
60
|
+
if parsed.scheme not in ("http", "https"):
|
|
61
|
+
raise ValueError("only http/https URLs are allowed")
|
|
62
|
+
if not parsed.hostname:
|
|
63
|
+
raise ValueError("URL has no host")
|
|
64
|
+
if allow_private:
|
|
65
|
+
return
|
|
66
|
+
try:
|
|
67
|
+
infos = socket.getaddrinfo(parsed.hostname, None)
|
|
68
|
+
except OSError as exc:
|
|
69
|
+
raise ValueError(f"cannot resolve host: {exc}") from exc
|
|
70
|
+
for info in infos:
|
|
71
|
+
ip = ipaddress.ip_address(info[4][0])
|
|
72
|
+
if (ip.is_private or ip.is_loopback or ip.is_link_local
|
|
73
|
+
or ip.is_reserved):
|
|
74
|
+
raise ValueError("private/loopback hosts are blocked")
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def fetch_url(url: str, allow_private: bool = False,
|
|
78
|
+
timeout: int = 15) -> tuple[str, str, str]:
|
|
79
|
+
_check_public(url, allow_private)
|
|
80
|
+
req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
|
|
81
|
+
with urllib.request.urlopen(req, timeout=timeout) as response:
|
|
82
|
+
ctype = response.headers.get("Content-Type", "")
|
|
83
|
+
raw = response.read(MAX_BYTES + 1)
|
|
84
|
+
final = response.geturl()
|
|
85
|
+
if len(raw) > MAX_BYTES:
|
|
86
|
+
raise ValueError("page is too large")
|
|
87
|
+
if "html" in ctype:
|
|
88
|
+
text = raw.decode("utf-8", errors="replace")
|
|
89
|
+
else:
|
|
90
|
+
text = raw.decode("utf-8", errors="replace")
|
|
91
|
+
return final, text, ctype
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def index_url(memory, url: str, title: str = "",
|
|
95
|
+
allow_private: bool = False) -> dict:
|
|
96
|
+
final, body, ctype = fetch_url(url, allow_private=allow_private)
|
|
97
|
+
if "html" in ctype:
|
|
98
|
+
parsed_title, text = html_to_text(body)
|
|
99
|
+
else:
|
|
100
|
+
parsed_title, text = "", body
|
|
101
|
+
text = text.strip()
|
|
102
|
+
if len(text) < 20:
|
|
103
|
+
raise ValueError("page has no usable text")
|
|
104
|
+
title = title or parsed_title or final
|
|
105
|
+
chunks = memory.add_text(final, text, title=title)
|
|
106
|
+
return {"source": final, "title": title, "chunks": chunks,
|
|
107
|
+
"preview": text[:400]}
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def wikipedia_research(memory, query: str, lang: str = "en",
|
|
111
|
+
limit: int = 2, timeout: int = 20,
|
|
112
|
+
intro_only: bool = True) -> list[dict]:
|
|
113
|
+
api = f"https://{lang}.wikipedia.org/w/api.php"
|
|
114
|
+
params = {
|
|
115
|
+
"action": "query", "list": "search", "srsearch": query,
|
|
116
|
+
"format": "json", "srlimit": max(3, min(10, limit * 2)),
|
|
117
|
+
}
|
|
118
|
+
req = urllib.request.Request(
|
|
119
|
+
api + "?" + urllib.parse.urlencode(params),
|
|
120
|
+
headers={"User-Agent": USER_AGENT})
|
|
121
|
+
with urllib.request.urlopen(req, timeout=timeout) as response:
|
|
122
|
+
data = json.loads(response.read().decode("utf-8"))
|
|
123
|
+
results = [result for result in
|
|
124
|
+
data.get("query", {}).get("search", [])
|
|
125
|
+
if result.get("pageid")]
|
|
126
|
+
wanted = query.strip().lower()
|
|
127
|
+
|
|
128
|
+
def rank(item):
|
|
129
|
+
index, result = item
|
|
130
|
+
title = str(result.get("title", "")).strip().lower()
|
|
131
|
+
if title == wanted:
|
|
132
|
+
return (0, index)
|
|
133
|
+
if title.startswith(wanted + " ("):
|
|
134
|
+
return (1, index)
|
|
135
|
+
if title.startswith(wanted):
|
|
136
|
+
return (2, index)
|
|
137
|
+
return (3, index)
|
|
138
|
+
|
|
139
|
+
results = [result for _, result in
|
|
140
|
+
sorted(enumerate(results), key=rank)][:max(1, min(limit, 5))]
|
|
141
|
+
pages = []
|
|
142
|
+
for result in results:
|
|
143
|
+
pageid = result.get("pageid")
|
|
144
|
+
title = result.get("title", "")
|
|
145
|
+
if not pageid:
|
|
146
|
+
continue
|
|
147
|
+
extract_params = {
|
|
148
|
+
"action": "query", "prop": "extracts", "explaintext": 1,
|
|
149
|
+
"exintro": 1 if intro_only else 0,
|
|
150
|
+
"pageids": pageid, "format": "json",
|
|
151
|
+
}
|
|
152
|
+
req = urllib.request.Request(
|
|
153
|
+
api + "?" + urllib.parse.urlencode(extract_params),
|
|
154
|
+
headers={"User-Agent": USER_AGENT})
|
|
155
|
+
with urllib.request.urlopen(req, timeout=timeout) as response:
|
|
156
|
+
extract = json.loads(response.read().decode("utf-8"))
|
|
157
|
+
page = extract.get("query", {}).get("pages", {}).get(str(pageid), {})
|
|
158
|
+
text = (page.get("extract") or "").strip()
|
|
159
|
+
if len(text) < 40:
|
|
160
|
+
continue
|
|
161
|
+
source = f"https://{lang}.wikipedia.org/?curid={pageid}"
|
|
162
|
+
chunks = memory.add_text(source, text, title=title)
|
|
163
|
+
pages.append({"source": source, "title": title, "chunks": chunks,
|
|
164
|
+
"preview": text[:400], "context": text[:4000]})
|
|
165
|
+
return pages
|