atomzero 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,2494 @@
1
+ """Local HTTP server for Atom Zero, the tiny transformer daily assistant.
2
+
3
+ python -m atomzero.chat_server \
4
+ --model checkpoints/transformer_behavior_50m_scaled \
5
+ --adapter checkpoints/adapters_50m/v0001 \
6
+ --docs data/docs.db
7
+
8
+ Endpoints:
9
+ GET /health
10
+ GET /v1/models
11
+ GET /v1/documents
12
+ GET /v1/documents/evidence?source=...&page=...&section=...
13
+ POST /v1/documents
14
+ POST /v1/documents/upload?source=...&title=...
15
+ POST /v1/documents/search
16
+ GET /v1/teacher/notes
17
+ POST /v1/teacher/notes
18
+ POST /v1/teacher/notes/verify
19
+ POST /v1/teacher/ask
20
+ POST /v1/teach
21
+ POST /v1/research
22
+ GET /v1/research/queue
23
+ POST /v1/research/queue
24
+ POST /v1/research/queue/approve
25
+ POST /v1/research/queue/reject
26
+ POST /v1/research/queue/run
27
+ POST /v1/learn_url
28
+ POST /v1/agent/chat
29
+ POST /v1/agent/plan
30
+ POST /generate
31
+ POST /v1/chat/completions
32
+ """
33
+ from __future__ import annotations
34
+
35
+ import argparse
36
+ import hashlib
37
+ import json
38
+ import os
39
+ import threading
40
+ import time
41
+ import urllib.parse
42
+ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
43
+ from pathlib import Path
44
+
45
+ import torch
46
+
47
+ from atomzero.doc_memory import (
48
+ SourceNotFound, add_citation_source, best_snippet, format_citation)
49
+ from atomzero.curiosity import (
50
+ CuriosityError, curiosity_items, propose_from_gaps)
51
+ from atomzero.feedback import (
52
+ FeedbackError, FeedbackNotFound, FeedbackStore)
53
+ from atomzero.knowledge_gaps import (
54
+ GapNotFound, KnowledgeGapError, KnowledgeGapStore)
55
+ from atomzero.neural_embedder import NeuralEmbedderError, NeuralReranker
56
+ from atomzero.research_queue import (
57
+ ItemNotFound, ResearchQueue, ResearchQueueError)
58
+ from atomzero.teacher_notes import TeacherNoteTooLarge, add_teacher_note
59
+ from atomzero.teacher_provider import TeacherProvider, TeacherProviderError
60
+ from atomzero.tool_planner import ToolPlanError, execute_plan
61
+ from atomzero.transformer import TransformerLM, generate
62
+
63
+ _lock = threading.Lock()
64
+ _state: dict = {}
65
+
66
+ MAX_BODY_BYTES = 2_000_000
67
+ MAX_DOC_BYTES = 10_000_000
68
+ MAX_SOURCE_CHARS = 512
69
+ MAX_TITLE_CHARS = 200
70
+
71
+
72
+ class DocumentTooLarge(ValueError):
73
+ pass
74
+
75
+
76
+ def ingest_document(docs, source, text, title: str = "") -> dict:
77
+ """Validate and index a text document, returning its provenance."""
78
+ if docs is None:
79
+ raise ValueError("no document memory configured")
80
+ source = str(source or "").strip()
81
+ title = str(title or "").strip()
82
+ if not source:
83
+ raise ValueError("source is required")
84
+ if len(source) > MAX_SOURCE_CHARS:
85
+ raise ValueError(f"source is too long (max {MAX_SOURCE_CHARS} chars)")
86
+ if len(title) > MAX_TITLE_CHARS:
87
+ raise ValueError(f"title is too long (max {MAX_TITLE_CHARS} chars)")
88
+ if any(ord(ch) < 32 for ch in source + title):
89
+ raise ValueError("source/title contains control characters")
90
+ if not isinstance(text, str):
91
+ raise ValueError("text must be a string")
92
+ text = text.strip()
93
+ if not text:
94
+ raise ValueError("text is empty")
95
+ raw = text.encode("utf-8")
96
+ if len(raw) > MAX_DOC_BYTES:
97
+ raise DocumentTooLarge(
98
+ f"document is too large (max {MAX_DOC_BYTES} bytes)")
99
+ chunks = docs.add_text(source, text, title=title)
100
+ content_sha = hashlib.sha256(raw).hexdigest()
101
+ return {
102
+ "source": source,
103
+ "title": title or Path(source).stem,
104
+ "chunks": chunks,
105
+ "bytes": len(raw),
106
+ "sha256": content_sha,
107
+ "content_sha256": content_sha,
108
+ "replace": True,
109
+ }
110
+
111
+
112
+ def ingest_upload(docs, source, data, title: str = "") -> dict:
113
+ """Validate and index a text/PDF/DOCX/HTML upload."""
114
+ if docs is None:
115
+ raise ValueError("no document memory configured")
116
+ source = str(source or "").strip()
117
+ title = str(title or "").strip()
118
+ if not source:
119
+ raise ValueError("source is required")
120
+ if len(source) > MAX_SOURCE_CHARS:
121
+ raise ValueError(f"source is too long (max {MAX_SOURCE_CHARS} chars)")
122
+ if len(title) > MAX_TITLE_CHARS:
123
+ raise ValueError(f"title is too long (max {MAX_TITLE_CHARS} chars)")
124
+ if any(ord(ch) < 32 for ch in source + title):
125
+ raise ValueError("source/title contains control characters")
126
+ if not isinstance(data, (bytes, bytearray)):
127
+ raise ValueError("document data must be bytes")
128
+ data = bytes(data)
129
+ if len(data) > MAX_DOC_BYTES:
130
+ raise DocumentTooLarge(
131
+ f"document is too large (max {MAX_DOC_BYTES} bytes)")
132
+ from atomzero.doc_ingest import (
133
+ DocumentExtractError, extract_bytes)
134
+
135
+ try:
136
+ extracted = extract_bytes(source, data, title=title)
137
+ except DocumentExtractError as exc:
138
+ raise ValueError(str(exc)) from exc
139
+ chunks = docs.add_blocks(
140
+ source, extracted["blocks"], title=extracted["title"])
141
+ pages = [block.get("page") for block in extracted["blocks"]
142
+ if block.get("page") is not None]
143
+ file_sha = hashlib.sha256(data).hexdigest()
144
+ content_text = "\n\n".join(
145
+ str(block.get("text", "")) for block in extracted["blocks"])
146
+ content_sha = hashlib.sha256(content_text.encode("utf-8")).hexdigest()
147
+ return {
148
+ "source": source,
149
+ "title": extracted["title"],
150
+ "chunks": chunks,
151
+ "bytes": len(data),
152
+ "sha256": file_sha,
153
+ "file_sha256": file_sha,
154
+ "content_sha256": content_sha,
155
+ "replace": True,
156
+ "format": extracted["format"],
157
+ "pages": len(set(pages)),
158
+ }
159
+
160
+
161
+ def resolve_adapter(adapter: str,
162
+ active_file: str = "data/active_adapter.json") -> str:
163
+ """Use the promoted adapter when ``adapter`` is ``auto``."""
164
+ if adapter != "auto":
165
+ return adapter
166
+ path = Path(active_file)
167
+ try:
168
+ payload = json.loads(path.read_text(encoding="utf-8"))
169
+ except (OSError, json.JSONDecodeError):
170
+ return ""
171
+ return str(payload.get("path", ""))
172
+
173
+
174
+ PAGE = """<!doctype html><meta charset="utf-8">
175
+ <meta name="viewport" content="width=device-width,initial-scale=1">
176
+ <title>Atom Zero - local assistant</title>
177
+ <style>
178
+ body{font:16px/1.5 system-ui;max-width:820px;margin:32px auto;padding:0 20px;color:#111}
179
+ h1{margin:0 0 4px}.hint{color:#666;font-size:14px}
180
+ #chat{display:flex;flex-direction:column;gap:10px;min-height:220px;margin:18px 0}
181
+ .msg{max-width:85%;padding:10px 12px;border-radius:10px;white-space:pre-wrap}
182
+ .user{align-self:flex-end;background:#e8f0fe}.bot{align-self:flex-start;background:#f4f4f4}
183
+ .meta{font-size:12px;color:#666;margin-top:4px}.err{background:#fde8e8}
184
+ .row{display:flex;gap:8px}.row input{flex:1;padding:10px;font:inherit}
185
+ textarea{width:100%;min-height:100px;padding:10px;font:inherit;box-sizing:border-box}
186
+ input,button{font:inherit}button{padding:8px 12px;cursor:pointer}
187
+ .chips{display:flex;flex-wrap:wrap;gap:6px;margin:8px 0 18px}
188
+ .chips button{font-size:13px;background:#fff;border:1px solid #ccc;border-radius:999px}
189
+ select{padding:8px;font:inherit;max-width:120px}
190
+ #rlimit{flex:0 0 70px;width:70px}
191
+ .row label{display:flex;align-items:center;gap:6px}
192
+ .qitem{border:1px solid #ddd;border-radius:8px;padding:10px;margin:8px 0}
193
+ .qitem .actions{display:flex;gap:8px;flex-wrap:wrap;margin-top:8px}
194
+ .qitem .src{font-size:12px;color:#555;word-break:break-all;margin-top:4px}
195
+ .msg .actions{display:flex;gap:6px;align-items:center;flex-wrap:wrap;margin-top:6px}
196
+ .msg .actions button{font-size:12px;padding:4px 8px}
197
+ hr{border:0;border-top:1px solid #ddd;margin:22px 0}
198
+ </style>
199
+ <h1>Atom Zero</h1>
200
+ <p class="hint">local-first personal assistant</p>
201
+ <p id="m" class="hint">loading...</p>
202
+ <div class="row"><button id="newsession">New session</button></div>
203
+ <div id="chat"></div>
204
+ <div class="row"><input id="p" placeholder="Ask, add a task, search your notes...">
205
+ <button id="g">Send</button></div>
206
+ <div class="chips">
207
+ <button data-q="what can you do?">what can it do</button>
208
+ <button data-q="add task buy milk">add task</button>
209
+ <button data-q="list tasks">list tasks</button>
210
+ <button data-q="my name is Alex">remember my name</button>
211
+ <button data-q="what do you remember?">recall</button>
212
+ <button data-q="what is 12 * 7?">calculate</button>
213
+ </div>
214
+ <hr>
215
+ <h3>Teach a fact</h3>
216
+ <p class="hint">Stored in the local document index with a source. Facts do not
217
+ change model weights.</p>
218
+ <div class="row"><input id="f" placeholder="The project codename is Blue Lantern.">
219
+ <button id="t">Teach fact</button></div>
220
+ <p id="s" class="hint"></p>
221
+ <hr>
222
+ <h3>Add a document</h3>
223
+ <p class="hint">Text, Markdown, HTML, PDF or DOCX (max 10 MB). PDFs keep page
224
+ numbers; DOCX/HTML/Markdown keep section headings. Stored with source, title
225
+ and SHA-256; re-adding the same source replaces it. Nothing is written to
226
+ weights.</p>
227
+ <div class="row"><input id="src" value="my-note.txt" placeholder="Source (required)">
228
+ <input id="ttl" placeholder="Title (optional)"></div>
229
+ <textarea id="txt" placeholder="Paste text or choose a file..."></textarea>
230
+ <div class="row"><input id="file" type="file"
231
+ accept=".txt,.md,.markdown,.html,.htm,.pdf,.docx,text/plain,text/html,application/pdf,application/vnd.openxmlformats-officedocument.wordprocessingml.document">
232
+ <button id="add">Add document</button></div>
233
+ <p id="ds" class="hint"></p>
234
+ <h3>Indexed documents</h3>
235
+ <div id="docs"></div>
236
+ <div class="row"><input id="docq" placeholder="Search indexed documents...">
237
+ <button id="docsearch">Search</button></div>
238
+ <p id="docmsg" class="hint"></p>
239
+ <div id="docresults"></div>
240
+ <div id="evidence"></div>
241
+ <hr>
242
+ <h3>Delete data</h3>
243
+ <p class="hint">Delete an indexed source or this browser session's memory. This
244
+ cannot be undone.</p>
245
+ <div class="row"><input id="forgetsrc" placeholder="Exact source, e.g. my-note.txt">
246
+ <button id="forgetdoc">Forget source</button>
247
+ <button id="forgetmem">Forget session memory</button></div>
248
+ <p id="fmsg" class="hint"></p>
249
+ <hr>
250
+ <h3>Session backup</h3>
251
+ <p class="hint">Export or replace this session's facts, tasks and transcript.
252
+ The file contains personal memory; keep it private.</p>
253
+ <div class="row"><button id="mexport">Export session</button>
254
+ <input id="mimport" type="file" accept="application/json">
255
+ <button id="mimportbtn">Import session</button></div>
256
+ <p id="bmsg" class="hint"></p>
257
+ <hr>
258
+ <h3>Add a teacher answer</h3>
259
+ <p class="hint">Paste an answer from ChatGPT or another assistant. It is stored
260
+ as a derived, unverified note with its source URL. It is not treated as a fact
261
+ and does not change model weights.</p>
262
+ <div class="row"><input id="tt" value="chatgpt-web" placeholder="Teacher">
263
+ <input id="tm" placeholder="Model (optional)"></div>
264
+ <div class="row"><input id="tp" placeholder="Topic/question">
265
+ <input id="tu" placeholder="Source URL (optional)"></div>
266
+ <div class="row"><input id="tq" placeholder="Question for the configured provider">
267
+ <button id="task" disabled>Ask provider</button></div>
268
+ <p id="tstatus" class="hint"></p>
269
+ <p id="tamsg" class="hint"></p>
270
+ <textarea id="ta" placeholder="Paste the teacher answer..."></textarea>
271
+ <button id="tnote">Save teacher note</button>
272
+ <p id="tmsg" class="hint"></p>
273
+ <div class="row"><input id="trev" value="local-reviewer" placeholder="Reviewer">
274
+ <button id="trefresh">Refresh teacher notes</button></div>
275
+ <p id="tvmsg" class="hint"></p>
276
+ <div id="tnotes"></div>
277
+ <hr>
278
+ <h3>Research queue</h3>
279
+ <p class="hint">A topic enters as pending and is not fetched. Approve it, then
280
+ run approved topics; only approved items touch Wikipedia.</p>
281
+ <div class="row">
282
+ <input id="rtopic" placeholder="Topic, e.g. Ankara">
283
+ <select id="rlang"><option value="en">en</option><option value="tr">tr</option></select>
284
+ <select id="rpages"><option value="1">1 page</option><option value="2" selected>2 pages</option><option value="3">3 pages</option></select>
285
+ <button id="radd">Add topic</button>
286
+ </div>
287
+ <div class="row">
288
+ <label>run limit <input id="rlimit" type="number" min="1" max="5" value="1"></label>
289
+ <button id="rrun">Run approved</button>
290
+ </div>
291
+ <p id="rmsg" class="hint"></p>
292
+ <div id="rq"></div>
293
+ <hr>
294
+ <h3>Learning queue</h3>
295
+ <p class="hint">Rate chat replies with Good / Bad. Approved examples are exported
296
+ for a gated LoRA update; nothing changes weights automatically.</p>
297
+ <div class="row"><button id="lrefresh">Refresh</button>
298
+ <button id="lexport">Export approved</button></div>
299
+ <p id="lmsg" class="hint"></p>
300
+ <div id="lq"></div>
301
+ <hr>
302
+ <h3>Adapter versions</h3>
303
+ <p class="hint">Accepted <b>vNNNN</b> versions and the active adapter. Rollback
304
+ is explicit CLI: <code>python research/learning_loop.py activate v0001</code>.</p>
305
+ <div class="row"><button id="arefresh">Refresh versions</button></div>
306
+ <p id="amsg" class="hint"></p>
307
+ <div id="aq"></div>
308
+ <hr>
309
+ <h3>Knowledge gaps</h3>
310
+ <p class="hint">When no local source answers a question, it is recorded here.
311
+ Propose one gap, or use <b>Propose curiosity topics</b> to create pending
312
+ research topics from up to three gaps. Approve them in the research queue
313
+ before the worker fetches sources or calls the teacher. Nothing changes
314
+ weights automatically.</p>
315
+ <div class="row"><button id="grefresh">Refresh gaps</button>
316
+ <button id="gcuriosity">Propose curiosity topics</button></div>
317
+ <p id="gmsg" class="hint"></p>
318
+ <div id="gq"></div>
319
+ <script>
320
+ function getSessionId() {
321
+ try {
322
+ const saved = localStorage.getItem('atomzero_session');
323
+ if (saved) return saved;
324
+ } catch (e) {}
325
+ const id = 'web-' + Math.random().toString(36).slice(2, 10);
326
+ try { localStorage.setItem('atomzero_session', id); } catch (e) {}
327
+ return id;
328
+ }
329
+ let sessionId = getSessionId();
330
+ document.getElementById('newsession').onclick = () => {
331
+ if (!confirm('Start a new session? The old session memory stays on disk until deleted.')) return;
332
+ try { localStorage.removeItem('atomzero_session'); } catch (e) {}
333
+ location.reload();
334
+ };
335
+ const chat = document.getElementById('chat');
336
+ function add(role, text, extra, isError) {
337
+ const box = document.createElement('div');
338
+ box.className = 'msg ' + role + (isError ? ' err' : '');
339
+ box.textContent = text;
340
+ if (extra) {
341
+ const meta = document.createElement('div');
342
+ meta.className = 'meta';
343
+ meta.textContent = extra;
344
+ box.appendChild(meta);
345
+ }
346
+ chat.appendChild(box);
347
+ box.scrollIntoView({block: 'end'});
348
+ return box;
349
+ }
350
+ async function restoreChat() {
351
+ try {
352
+ const r = await fetch('/v1/memory/history', {
353
+ method: 'POST', headers: {'Content-Type': 'application/json'},
354
+ body: JSON.stringify({session_id: sessionId, limit: 50})});
355
+ const j = await r.json();
356
+ if (!r.ok) return;
357
+ (j.data || []).forEach(item => {
358
+ const role = item.role === 'assistant' ? 'bot' : 'user';
359
+ add(role, item.text || '');
360
+ });
361
+ } catch (e) {}
362
+ }
363
+ function extra(result) {
364
+ const parts = [];
365
+ const tools = result.tool_calls || [];
366
+ if (tools.length) parts.push('tools: ' + tools.map(t => t.name).join(', '));
367
+ const sources = result.sources || [];
368
+ if (sources.length) parts.push('sources: ' + sources.map(s => {
369
+ let status = '';
370
+ if (s.kind === 'teacher') {
371
+ status = s.verified
372
+ ? (s.reviewer ? ' [verified: ' + s.reviewer + ']' : ' [verified]')
373
+ : ' [unverified]';
374
+ }
375
+ return s.title + ' (' + (s.source_url || s.source) + ')' + status;
376
+ }).join('; '));
377
+ return parts.join(' | ');
378
+ }
379
+ async function send(message) {
380
+ const text = (message || document.getElementById('p').value).trim();
381
+ if (!text) return;
382
+ document.getElementById('p').value = '';
383
+ add('user', text);
384
+ try {
385
+ const r = await fetch('/v1/agent/chat', {method: 'POST',
386
+ headers: {'Content-Type': 'application/json'},
387
+ body: JSON.stringify({session_id: sessionId, message: text})});
388
+ const j = await r.json();
389
+ if (!r.ok) return add('bot', 'error: ' + (j.error || r.status), '', true);
390
+ const box = add('bot', j.reply, extra(j));
391
+ addFeedback(box, text, j.reply);
392
+ refreshGaps();
393
+ } catch (e) {
394
+ add('bot', 'error: ' + e, '', true);
395
+ }
396
+ }
397
+ function addFeedback(box, user, assistant) {
398
+ const row = document.createElement('div');
399
+ row.className = 'actions';
400
+ const status = document.createElement('span');
401
+ status.className = 'hint';
402
+ const good = document.createElement('button');
403
+ good.textContent = 'Good';
404
+ good.onclick = () => sendFeedback(user, assistant, 1, '', status);
405
+ const bad = document.createElement('button');
406
+ bad.textContent = 'Bad / correct';
407
+ bad.onclick = () => {
408
+ const corrected = prompt('Correct answer (optional):', '');
409
+ sendFeedback(user, assistant, -1, corrected || '', status);
410
+ };
411
+ row.appendChild(good);
412
+ row.appendChild(bad);
413
+ row.appendChild(status);
414
+ box.appendChild(row);
415
+ }
416
+ async function sendFeedback(user, assistant, rating, corrected, status) {
417
+ status.textContent = 'saving...';
418
+ try {
419
+ const r = await fetch('/v1/learning/feedback', {
420
+ method: 'POST', headers: {'Content-Type': 'application/json'},
421
+ body: JSON.stringify({user, assistant, rating, corrected,
422
+ session_id: sessionId})});
423
+ const j = await r.json();
424
+ status.textContent = r.ok ? 'feedback queued'
425
+ : ('error: ' + (j.error || r.status));
426
+ if (r.ok) refreshLearning();
427
+ } catch (e) {
428
+ status.textContent = 'error: ' + e;
429
+ }
430
+ }
431
+ document.getElementById('g').onclick = () => send();
432
+ document.getElementById('p').addEventListener('keydown', e => {
433
+ if (e.key === 'Enter') send();
434
+ });
435
+ document.querySelectorAll('.chips button').forEach(b => {
436
+ b.onclick = () => send(b.dataset.q);
437
+ });
438
+ document.getElementById('t').onclick = async () => {
439
+ const fact = document.getElementById('f').value.trim();
440
+ if (!fact) return;
441
+ const out = document.getElementById('s');
442
+ try {
443
+ const r = await fetch('/v1/teach', {method: 'POST',
444
+ headers: {'Content-Type': 'application/json'},
445
+ body: JSON.stringify({fact})});
446
+ const j = await r.json();
447
+ out.textContent = r.ok ? ('Saved ' + j.fact_chunks + ' chunk(s) to local memory.')
448
+ : ('error: ' + (j.error || r.status));
449
+ if (r.ok) {
450
+ document.getElementById('f').value = '';
451
+ refreshHealth();
452
+ }
453
+ } catch (e) { out.textContent = 'error: ' + e; }
454
+ };
455
+ let selectedFile = null;
456
+ document.getElementById('txt').addEventListener('input', () => {
457
+ selectedFile = null;
458
+ });
459
+ document.getElementById('file').onchange = () => {
460
+ const file = document.getElementById('file').files[0];
461
+ const out = document.getElementById('ds');
462
+ selectedFile = file || null;
463
+ if (!file) return;
464
+ if (file.size > 10000000) {
465
+ out.textContent = 'file is too large (max 10 MB)';
466
+ selectedFile = null;
467
+ return;
468
+ }
469
+ const src = document.getElementById('src');
470
+ if (!src.value.trim() || src.value === 'my-note.txt') src.value = file.name;
471
+ const title = document.getElementById('ttl');
472
+ if (!title.value.trim()) title.value = file.name;
473
+ const textLike = /\\.(txt|md|markdown|html?)$/i.test(file.name);
474
+ if (textLike) {
475
+ const reader = new FileReader();
476
+ reader.onload = () => {
477
+ document.getElementById('txt').value = reader.result;
478
+ out.textContent = 'Selected ' + file.name + ' - click Add document.';
479
+ };
480
+ reader.readAsText(file);
481
+ } else {
482
+ document.getElementById('txt').value = '';
483
+ out.textContent = 'Selected ' + file.name + ' - click Add document.';
484
+ }
485
+ };
486
+ document.getElementById('add').onclick = async () => {
487
+ const source = document.getElementById('src').value.trim();
488
+ const text = document.getElementById('txt').value;
489
+ const out = document.getElementById('ds');
490
+ if (!source) {
491
+ out.textContent = 'source is required';
492
+ return;
493
+ }
494
+ if (!selectedFile && !text.trim()) {
495
+ out.textContent = 'source and text (or a file) are required';
496
+ return;
497
+ }
498
+ try {
499
+ let r;
500
+ if (selectedFile) {
501
+ const query = new URLSearchParams({
502
+ source, title: document.getElementById('ttl').value});
503
+ r = await fetch('/v1/documents/upload?' + query.toString(), {
504
+ method: 'POST',
505
+ headers: {'Content-Type': 'application/octet-stream'},
506
+ body: selectedFile});
507
+ } else {
508
+ r = await fetch('/v1/documents', {method: 'POST',
509
+ headers: {'Content-Type': 'application/json'},
510
+ body: JSON.stringify({source,
511
+ title: document.getElementById('ttl').value,
512
+ text})});
513
+ }
514
+ const j = await r.json();
515
+ out.textContent = r.ok
516
+ ? ('Indexed ' + j.chunks + ' chunk(s) from ' + j.source +
517
+ (j.format ? ' [' + j.format + ']' : '') +
518
+ ' | sha256 ' + j.sha256.slice(0, 12) + '...')
519
+ : ('error: ' + (j.error || r.status));
520
+ if (r.ok) {
521
+ document.getElementById('txt').value = '';
522
+ document.getElementById('file').value = '';
523
+ selectedFile = null;
524
+ refreshHealth();
525
+ refreshDocs();
526
+ }
527
+ } catch (e) { out.textContent = 'error: ' + e; }
528
+ };
529
+ async function refreshDocs() {
530
+ const box = document.getElementById('docs');
531
+ if (!box) return;
532
+ try {
533
+ const r = await fetch('/v1/documents');
534
+ const j = await r.json();
535
+ box.textContent = '';
536
+ if (!r.ok) {
537
+ const error = document.createElement('div');
538
+ error.className = 'hint';
539
+ error.textContent = 'document error: ' + (j.error || r.status);
540
+ box.appendChild(error);
541
+ return;
542
+ }
543
+ const data = j.data || [];
544
+ if (!data.length) {
545
+ const empty = document.createElement('div');
546
+ empty.className = 'hint';
547
+ empty.textContent = 'No indexed documents.';
548
+ box.appendChild(empty);
549
+ return;
550
+ }
551
+ data.forEach(item => {
552
+ const card = document.createElement('div');
553
+ card.className = 'qitem';
554
+ const title = document.createElement('div');
555
+ const pageInfo = item.page_start
556
+ ? ' | pp. ' + item.page_start + '-' + item.page_end : '';
557
+ title.textContent = (item.title || item.source) + ' | ' +
558
+ (item.chunks || 0) + ' chunk(s)' + pageInfo;
559
+ card.appendChild(title);
560
+ const source = document.createElement('div');
561
+ source.className = 'src';
562
+ source.textContent = item.source + (item.kind && item.kind !== 'document'
563
+ ? ' [' + item.kind + ']' : '');
564
+ card.appendChild(source);
565
+ const actions = document.createElement('div');
566
+ actions.className = 'actions';
567
+ actions.appendChild(qbutton('Evidence', () => showEvidence(item.source)));
568
+ actions.appendChild(qbutton('Forget', () => forgetSource(item.source)));
569
+ card.appendChild(actions);
570
+ box.appendChild(card);
571
+ });
572
+ } catch (e) {
573
+ box.textContent = 'document error: ' + e;
574
+ }
575
+ }
576
+ async function searchDocs() {
577
+ const query = document.getElementById('docq').value.trim();
578
+ const msg = document.getElementById('docmsg');
579
+ const box = document.getElementById('docresults');
580
+ box.textContent = '';
581
+ document.getElementById('evidence').textContent = '';
582
+ if (!query) { msg.textContent = 'enter a search query'; return; }
583
+ msg.textContent = 'searching...';
584
+ try {
585
+ const r = await fetch('/v1/documents/search', {
586
+ method: 'POST', headers: {'Content-Type': 'application/json'},
587
+ body: JSON.stringify({query, k: 5})});
588
+ const j = await r.json();
589
+ if (!r.ok) { msg.textContent = 'error: ' + (j.error || r.status); return; }
590
+ const hits = j.data || [];
591
+ msg.textContent = hits.length + ' hit(s)';
592
+ hits.forEach(hit => {
593
+ const card = document.createElement('div');
594
+ card.className = 'qitem';
595
+ const title = document.createElement('div');
596
+ const where = (hit.page ? ' | p. ' + hit.page : '') +
597
+ (hit.section ? ' | ' + hit.section : '');
598
+ title.textContent = (hit.title || hit.source) + ' | ' + hit.source +
599
+ where;
600
+ card.appendChild(title);
601
+ const snippet = document.createElement('div');
602
+ snippet.className = 'src';
603
+ snippet.textContent = (hit.chunk || '').slice(0, 400);
604
+ card.appendChild(snippet);
605
+ const actions = document.createElement('div');
606
+ actions.className = 'actions';
607
+ actions.appendChild(qbutton('Evidence', () =>
608
+ showEvidence(hit.source, hit.page, hit.section)));
609
+ card.appendChild(actions);
610
+ box.appendChild(card);
611
+ });
612
+ } catch (e) { msg.textContent = 'error: ' + e; }
613
+ }
614
+ async function showEvidence(source, page, section) {
615
+ const box = document.getElementById('evidence');
616
+ box.textContent = 'loading evidence...';
617
+ const query = new URLSearchParams({source});
618
+ if (page) query.set('page', page);
619
+ if (section) query.set('section', section);
620
+ try {
621
+ const r = await fetch('/v1/documents/evidence?' + query.toString());
622
+ const j = await r.json();
623
+ box.textContent = '';
624
+ if (!r.ok) {
625
+ box.textContent = 'evidence error: ' + (j.error || r.status);
626
+ return;
627
+ }
628
+ const head = document.createElement('div');
629
+ head.className = 'hint';
630
+ head.textContent = 'Evidence: ' + source +
631
+ (page ? ' | p. ' + page : '') +
632
+ (section ? ' | ' + section : '') +
633
+ ' | ' + (j.returned || 0) + ' chunk(s)';
634
+ box.appendChild(head);
635
+ (j.data || []).forEach(item => {
636
+ const card = document.createElement('div');
637
+ card.className = 'qitem';
638
+ const title = document.createElement('div');
639
+ title.textContent = (item.page ? 'p. ' + item.page : '') +
640
+ (item.section ? (item.page ? ' | ' : '') + item.section : '');
641
+ if (title.textContent) card.appendChild(title);
642
+ const text = document.createElement('div');
643
+ text.className = 'src';
644
+ text.textContent = item.chunk || '';
645
+ card.appendChild(text);
646
+ box.appendChild(card);
647
+ });
648
+ } catch (e) { box.textContent = 'evidence error: ' + e; }
649
+ }
650
+ document.getElementById('docsearch').onclick = () => searchDocs();
651
+ document.getElementById('docq').addEventListener('keydown', event => {
652
+ if (event.key === 'Enter') searchDocs();
653
+ });
654
+ async function forgetSource(source) {
655
+ if (!confirm('Delete indexed source "' + source + '"?')) return;
656
+ try {
657
+ const r = await fetch('/v1/documents/forget', {
658
+ method: 'POST', headers: {'Content-Type': 'application/json'},
659
+ body: JSON.stringify({source})});
660
+ const j = await r.json();
661
+ setForgetMsg(r.ok
662
+ ? ('deleted ' + j.chunks_deleted + ' chunk(s) from ' + source)
663
+ : ('error: ' + (j.error || r.status)), !r.ok);
664
+ if (r.ok) { refreshHealth(); refreshDocs(); }
665
+ } catch (e) { setForgetMsg('error: ' + e, true); }
666
+ }
667
+ document.getElementById('tnote').onclick = async () => {
668
+ const answer = document.getElementById('ta').value.trim();
669
+ const out = document.getElementById('tmsg');
670
+ if (!answer) {
671
+ out.textContent = 'answer is required';
672
+ return;
673
+ }
674
+ out.textContent = 'saving...';
675
+ try {
676
+ const r = await fetch('/v1/teacher/notes', {
677
+ method: 'POST', headers: {'Content-Type': 'application/json'},
678
+ body: JSON.stringify({
679
+ teacher: document.getElementById('tt').value,
680
+ model: document.getElementById('tm').value,
681
+ topic: document.getElementById('tp').value,
682
+ source_url: document.getElementById('tu').value,
683
+ answer,
684
+ })});
685
+ const j = await r.json();
686
+ out.textContent = r.ok
687
+ ? ('Saved as unverified teacher note: ' + j.source +
688
+ ' (' + j.chunks + ' chunk(s))')
689
+ : ('error: ' + (j.error || r.status));
690
+ if (r.ok) {
691
+ document.getElementById('ta').value = '';
692
+ document.getElementById('tp').value = '';
693
+ document.getElementById('tu').value = '';
694
+ refreshHealth();
695
+ refreshTeacherNotes();
696
+ }
697
+ } catch (e) { out.textContent = 'error: ' + e; }
698
+ };
699
+ document.getElementById('task').onclick = async () => {
700
+ const prompt = document.getElementById('tq').value.trim();
701
+ const out = document.getElementById('tamsg');
702
+ if (!prompt) {
703
+ out.textContent = 'question is required';
704
+ return;
705
+ }
706
+ const button = document.getElementById('task');
707
+ button.disabled = true;
708
+ out.textContent = 'asking provider...';
709
+ try {
710
+ const r = await fetch('/v1/teacher/ask', {
711
+ method: 'POST', headers: {'Content-Type': 'application/json'},
712
+ body: JSON.stringify({
713
+ prompt,
714
+ topic: document.getElementById('tp').value,
715
+ source_url: document.getElementById('tu').value,
716
+ max_tokens: 512,
717
+ })});
718
+ const j = await r.json();
719
+ out.textContent = r.ok
720
+ ? ('Saved as unverified teacher note: ' + j.source +
721
+ ' (' + j.chunks + ' chunk(s))')
722
+ : ('error: ' + (j.error || r.status));
723
+ if (r.ok) {
724
+ document.getElementById('tq').value = '';
725
+ refreshHealth();
726
+ refreshTeacherNotes();
727
+ }
728
+ } catch (e) {
729
+ out.textContent = 'error: ' + e;
730
+ } finally {
731
+ button.disabled = false;
732
+ }
733
+ };
734
+ async function refreshTeacherNotes() {
735
+ const box = document.getElementById('tnotes');
736
+ try {
737
+ const r = await fetch('/v1/teacher/notes');
738
+ const j = await r.json();
739
+ if (!r.ok) {
740
+ box.textContent = 'teacher notes error: ' + (j.error || r.status);
741
+ return;
742
+ }
743
+ renderTeacherNotes(j.data || [], j.stats || {});
744
+ } catch (e) {
745
+ box.textContent = 'teacher notes error: ' + e;
746
+ }
747
+ }
748
+ function renderTeacherNotes(data, stats) {
749
+ const box = document.getElementById('tnotes');
750
+ box.textContent = '';
751
+ const summary = document.createElement('div');
752
+ summary.className = 'hint';
753
+ summary.textContent =
754
+ 'teacher notes: ' + (stats.total || 0) + ' total, ' +
755
+ (stats.unverified || 0) + ' unverified, ' +
756
+ (stats.verified || 0) + ' verified';
757
+ box.appendChild(summary);
758
+ if (!data.length) {
759
+ const empty = document.createElement('div');
760
+ empty.className = 'hint';
761
+ empty.textContent = 'No teacher notes yet.';
762
+ box.appendChild(empty);
763
+ return;
764
+ }
765
+ data.forEach(note => {
766
+ const card = document.createElement('div');
767
+ card.className = 'qitem';
768
+ const title = document.createElement('div');
769
+ title.textContent = (note.verified ? 'VERIFIED' : 'UNVERIFIED') +
770
+ ' | ' + (note.title || note.source);
771
+ card.appendChild(title);
772
+ const meta = document.createElement('div');
773
+ meta.className = 'src';
774
+ meta.textContent = 'teacher: ' + (note.teacher || '') +
775
+ ' / ' + (note.model || '') +
776
+ ' | ' + (note.source || '');
777
+ card.appendChild(meta);
778
+ if (note.source_url) {
779
+ const url = document.createElement('div');
780
+ url.className = 'src';
781
+ url.textContent = 'source_url: ' + note.source_url;
782
+ card.appendChild(url);
783
+ }
784
+ if (note.verified) {
785
+ const reviewed = document.createElement('div');
786
+ reviewed.className = 'src';
787
+ reviewed.textContent = 'reviewed by ' + (note.reviewer || 'unknown') +
788
+ ' at ' + (note.verified_at || 'unknown');
789
+ card.appendChild(reviewed);
790
+ } else {
791
+ const actions = document.createElement('div');
792
+ actions.className = 'actions';
793
+ actions.appendChild(qbutton('Verify',
794
+ () => verifyTeacherNote(note.source)));
795
+ card.appendChild(actions);
796
+ }
797
+ box.appendChild(card);
798
+ });
799
+ }
800
+ async function verifyTeacherNote(source) {
801
+ const reviewer = document.getElementById('trev').value.trim();
802
+ const out = document.getElementById('tvmsg');
803
+ if (!reviewer) {
804
+ out.textContent = 'reviewer is required';
805
+ return;
806
+ }
807
+ out.textContent = 'verifying...';
808
+ try {
809
+ const r = await fetch('/v1/teacher/notes/verify', {
810
+ method: 'POST', headers: {'Content-Type': 'application/json'},
811
+ body: JSON.stringify({source, reviewer})});
812
+ const j = await r.json();
813
+ out.textContent = r.ok
814
+ ? ('Verified ' + j.source + ' by ' + j.reviewer)
815
+ : ('error: ' + (j.error || r.status));
816
+ if (r.ok) {
817
+ await refreshTeacherNotes();
818
+ refreshHealth();
819
+ }
820
+ } catch (e) {
821
+ out.textContent = 'error: ' + e;
822
+ }
823
+ }
824
+ document.getElementById('trefresh').onclick = () => refreshTeacherNotes();
825
+ function qmsg(text, isError) {
826
+ const el = document.getElementById('rmsg');
827
+ el.textContent = text;
828
+ el.style.color = isError ? '#b00020' : '#666';
829
+ }
830
+ function qbutton(label, fn) {
831
+ const button = document.createElement('button');
832
+ button.textContent = label;
833
+ button.onclick = fn;
834
+ return button;
835
+ }
836
+ function renderQueue(data, stats) {
837
+ const box = document.getElementById('rq');
838
+ box.textContent = '';
839
+ const summary = document.createElement('div');
840
+ summary.className = 'hint';
841
+ summary.textContent =
842
+ 'total ' + (stats.total || 0) +
843
+ ' | pending ' + (stats.pending || 0) +
844
+ ' | approved ' + (stats.approved || 0) +
845
+ ' | done ' + (stats.done || 0) +
846
+ ' | error ' + (stats.error || 0) +
847
+ ' | rejected ' + (stats.rejected || 0);
848
+ box.appendChild(summary);
849
+ if (!data.length) {
850
+ const empty = document.createElement('div');
851
+ empty.className = 'hint';
852
+ empty.textContent = 'No topics yet.';
853
+ box.appendChild(empty);
854
+ return;
855
+ }
856
+ data.forEach(item => {
857
+ const card = document.createElement('div');
858
+ card.className = 'qitem';
859
+ const title = document.createElement('div');
860
+ title.textContent = item.id + ' | ' + item.status +
861
+ (item.kind === 'curiosity' ? ' | curiosity' : '') + ' | ' + item.topic;
862
+ card.appendChild(title);
863
+ const meta = document.createElement('div');
864
+ meta.className = 'src';
865
+ meta.textContent = (item.lang || 'en') + ' | max pages ' +
866
+ (item.max_pages || 1);
867
+ card.appendChild(meta);
868
+ if (item.error) {
869
+ const error = document.createElement('div');
870
+ error.className = 'src';
871
+ error.textContent = 'error: ' + item.error;
872
+ card.appendChild(error);
873
+ }
874
+ (item.sources || []).forEach(source => {
875
+ const line = document.createElement('div');
876
+ line.className = 'src';
877
+ line.textContent = 'source: ' + source.title + ' | ' +
878
+ source.chunks + ' chunks | ' + source.source;
879
+ card.appendChild(line);
880
+ });
881
+ if (item.teacher && (item.teacher.source || item.teacher.error)) {
882
+ const line = document.createElement('div');
883
+ line.className = 'src';
884
+ const teacherStatus = item.teacher.error
885
+ ? item.teacher.error
886
+ : (item.teacher.verified
887
+ ? (item.teacher.reviewer
888
+ ? 'verified by ' + item.teacher.reviewer
889
+ : 'verified')
890
+ : 'unverified');
891
+ line.textContent = 'teacher: ' +
892
+ (item.teacher.source || 'no note') + ' | ' + teacherStatus;
893
+ card.appendChild(line);
894
+ if (item.teacher.answer_preview) {
895
+ const preview = document.createElement('div');
896
+ preview.className = 'src';
897
+ preview.textContent = 'synthesis: ' + item.teacher.answer_preview;
898
+ card.appendChild(preview);
899
+ }
900
+ }
901
+ if (item.status === 'done' && !(item.sources || []).length) {
902
+ const none = document.createElement('div');
903
+ none.className = 'src';
904
+ none.textContent = 'no pages found';
905
+ card.appendChild(none);
906
+ }
907
+ if (item.status === 'pending' || item.status === 'approved') {
908
+ const actions = document.createElement('div');
909
+ actions.className = 'actions';
910
+ if (item.status === 'pending') {
911
+ actions.appendChild(qbutton('Approve',
912
+ () => queueAction(item.id, 'approve')));
913
+ }
914
+ actions.appendChild(qbutton('Reject',
915
+ () => queueAction(item.id, 'reject')));
916
+ card.appendChild(actions);
917
+ }
918
+ box.appendChild(card);
919
+ });
920
+ }
921
+ async function refreshQueue() {
922
+ try {
923
+ const r = await fetch('/v1/research/queue');
924
+ const j = await r.json();
925
+ if (!r.ok) {
926
+ qmsg('queue error: ' + (j.error || r.status), true);
927
+ return;
928
+ }
929
+ renderQueue(j.data || [], j.stats || {});
930
+ } catch (e) {
931
+ qmsg('queue error: ' + e, true);
932
+ }
933
+ }
934
+ async function queueAction(id, action) {
935
+ qmsg(action + ' ' + id + '...');
936
+ try {
937
+ const r = await fetch('/v1/research/queue/' + action, {
938
+ method: 'POST', headers: {'Content-Type': 'application/json'},
939
+ body: JSON.stringify({id})});
940
+ const j = await r.json();
941
+ qmsg(r.ok ? (id + ' -> ' + j.status)
942
+ : ('error: ' + (j.error || r.status)), !r.ok);
943
+ await refreshQueue();
944
+ } catch (e) {
945
+ qmsg('error: ' + e, true);
946
+ }
947
+ }
948
+ document.getElementById('radd').onclick = async () => {
949
+ const topic = document.getElementById('rtopic').value.trim();
950
+ if (!topic) {
951
+ qmsg('topic is required', true);
952
+ return;
953
+ }
954
+ qmsg('adding ' + topic + '...');
955
+ try {
956
+ const r = await fetch('/v1/research/queue', {
957
+ method: 'POST', headers: {'Content-Type': 'application/json'},
958
+ body: JSON.stringify({
959
+ topic,
960
+ lang: document.getElementById('rlang').value,
961
+ pages: parseInt(document.getElementById('rpages').value, 10),
962
+ })});
963
+ const j = await r.json();
964
+ qmsg(r.ok ? ('Added ' + j.id + ' as pending.')
965
+ : ('error: ' + (j.error || r.status)), !r.ok);
966
+ if (r.ok) {
967
+ document.getElementById('rtopic').value = '';
968
+ await refreshQueue();
969
+ }
970
+ } catch (e) {
971
+ qmsg('error: ' + e, true);
972
+ }
973
+ };
974
+ document.getElementById('rrun').onclick = async () => {
975
+ const button = document.getElementById('rrun');
976
+ const limit = parseInt(document.getElementById('rlimit').value, 10) || 1;
977
+ const label = button.textContent;
978
+ button.disabled = true;
979
+ button.textContent = 'Fetching...';
980
+ qmsg('running approved topics...');
981
+ try {
982
+ const r = await fetch('/v1/research/queue/run', {
983
+ method: 'POST', headers: {'Content-Type': 'application/json'},
984
+ body: JSON.stringify({limit})});
985
+ const j = await r.json();
986
+ if (!r.ok) {
987
+ qmsg('run error: ' + (j.error || r.status), true);
988
+ } else {
989
+ const ran = j.ran || [];
990
+ const done = ran.filter(item => item.status === 'done').length;
991
+ const errors = ran.filter(item => item.status === 'error').length;
992
+ qmsg('processed ' + ran.length + ' item(s): ' +
993
+ done + ' done, ' + errors + ' error.');
994
+ }
995
+ await refreshQueue();
996
+ } catch (e) {
997
+ qmsg('run error: ' + e, true);
998
+ } finally {
999
+ button.disabled = false;
1000
+ button.textContent = label;
1001
+ }
1002
+ };
1003
+ function setForgetMsg(text, isError) {
1004
+ const el = document.getElementById('fmsg');
1005
+ el.textContent = text;
1006
+ el.style.color = isError ? '#b00020' : '#666';
1007
+ }
1008
+ document.getElementById('forgetdoc').onclick = async () => {
1009
+ const source = document.getElementById('forgetsrc').value.trim();
1010
+ if (!source) return setForgetMsg('Enter the exact source to delete.', true);
1011
+ if (!confirm('Delete indexed source "' + source + '"?')) return;
1012
+ try {
1013
+ const r = await fetch('/v1/documents/forget', {
1014
+ method: 'POST', headers: {'Content-Type': 'application/json'},
1015
+ body: JSON.stringify({source})});
1016
+ const j = await r.json();
1017
+ setForgetMsg(r.ok
1018
+ ? ('deleted ' + j.chunks_deleted + ' chunk(s) and ' +
1019
+ j.meta_deleted + ' metadata row(s)')
1020
+ : ('error: ' + (j.error || r.status)), !r.ok);
1021
+ if (r.ok) { refreshHealth(); refreshDocs(); }
1022
+ } catch (e) { setForgetMsg('error: ' + e, true); }
1023
+ };
1024
+ document.getElementById('forgetmem').onclick = async () => {
1025
+ if (!confirm('Delete this session memory (facts, tasks, episodes)?')) return;
1026
+ try {
1027
+ const r = await fetch('/v1/memory/forget', {
1028
+ method: 'POST', headers: {'Content-Type': 'application/json'},
1029
+ body: JSON.stringify({session_id: sessionId, all: true})});
1030
+ const j = await r.json();
1031
+ setForgetMsg(r.ok
1032
+ ? ('deleted facts ' + j.facts_deleted + ', episodes ' +
1033
+ j.episodes_deleted + ', tasks ' + j.tasks_deleted)
1034
+ : ('error: ' + (j.error || r.status)), !r.ok);
1035
+ } catch (e) { setForgetMsg('error: ' + e, true); }
1036
+ };
1037
+ function setLearningMsg(text, isError) {
1038
+ const el = document.getElementById('lmsg');
1039
+ el.textContent = text;
1040
+ el.style.color = isError ? '#b00020' : '#666';
1041
+ }
1042
+ function renderLearning(data, stats) {
1043
+ const box = document.getElementById('lq');
1044
+ box.textContent = '';
1045
+ const summary = document.createElement('div');
1046
+ summary.className = 'hint';
1047
+ summary.textContent =
1048
+ 'feedback: ' + (stats.total || 0) + ' total | pending ' +
1049
+ (stats.pending || 0) + ' | approved ' + (stats.approved || 0) +
1050
+ ' | rejected ' + (stats.rejected || 0);
1051
+ box.appendChild(summary);
1052
+ if (!data.length) {
1053
+ const empty = document.createElement('div');
1054
+ empty.className = 'hint';
1055
+ empty.textContent = 'No feedback yet. Rate a chat reply above.';
1056
+ box.appendChild(empty);
1057
+ return;
1058
+ }
1059
+ data.forEach(item => {
1060
+ const card = document.createElement('div');
1061
+ card.className = 'qitem';
1062
+ const title = document.createElement('div');
1063
+ title.textContent = item.id + ' | ' + item.status +
1064
+ ' | rating ' + item.rating;
1065
+ card.appendChild(title);
1066
+ const user = document.createElement('div');
1067
+ user.className = 'src';
1068
+ user.textContent = 'user: ' + (item.user || '').slice(0, 240);
1069
+ card.appendChild(user);
1070
+ const assistant = document.createElement('div');
1071
+ assistant.className = 'src';
1072
+ assistant.textContent = 'assistant: ' + (item.assistant || '').slice(0, 240);
1073
+ card.appendChild(assistant);
1074
+ if (item.corrected) {
1075
+ const corrected = document.createElement('div');
1076
+ corrected.className = 'src';
1077
+ corrected.textContent = 'corrected: ' + item.corrected.slice(0, 240);
1078
+ card.appendChild(corrected);
1079
+ }
1080
+ if (item.status === 'pending' || item.status === 'approved') {
1081
+ const actions = document.createElement('div');
1082
+ actions.className = 'actions';
1083
+ if (item.status === 'pending') {
1084
+ actions.appendChild(qbutton('Approve',
1085
+ () => learningAction(item.id, 'approve')));
1086
+ }
1087
+ actions.appendChild(qbutton('Reject',
1088
+ () => learningAction(item.id, 'reject')));
1089
+ card.appendChild(actions);
1090
+ }
1091
+ box.appendChild(card);
1092
+ });
1093
+ }
1094
+ async function refreshLearning() {
1095
+ try {
1096
+ const r = await fetch('/v1/learning/queue');
1097
+ const j = await r.json();
1098
+ if (!r.ok) {
1099
+ setLearningMsg('learning error: ' + (j.error || r.status), true);
1100
+ return;
1101
+ }
1102
+ renderLearning(j.data || [], j.stats || {});
1103
+ } catch (e) {
1104
+ setLearningMsg('learning error: ' + e, true);
1105
+ }
1106
+ }
1107
+ async function learningAction(id, action) {
1108
+ setLearningMsg(action + ' ' + id + '...');
1109
+ try {
1110
+ const r = await fetch('/v1/learning/queue/' + action, {
1111
+ method: 'POST', headers: {'Content-Type': 'application/json'},
1112
+ body: JSON.stringify({id})});
1113
+ const j = await r.json();
1114
+ setLearningMsg(r.ok ? (id + ' -> ' + j.status)
1115
+ : ('error: ' + (j.error || r.status)), !r.ok);
1116
+ await refreshLearning();
1117
+ } catch (e) {
1118
+ setLearningMsg('error: ' + e, true);
1119
+ }
1120
+ }
1121
+ document.getElementById('lrefresh').onclick = () => refreshLearning();
1122
+ document.getElementById('lexport').onclick = async () => {
1123
+ setLearningMsg('exporting...');
1124
+ try {
1125
+ const r = await fetch('/v1/learning/queue/export', {
1126
+ method: 'POST', headers: {'Content-Type': 'application/json'},
1127
+ body: '{}'});
1128
+ const j = await r.json();
1129
+ setLearningMsg(r.ok
1130
+ ? ('exported ' + j.examples + ' example(s) to ' + j.path)
1131
+ : ('error: ' + (j.error || r.status)), !r.ok);
1132
+ } catch (e) {
1133
+ setLearningMsg('error: ' + e, true);
1134
+ }
1135
+ };
1136
+ function setBackupMsg(text, isError) {
1137
+ const el = document.getElementById('bmsg');
1138
+ el.textContent = text;
1139
+ el.style.color = isError ? '#b00020' : '#666';
1140
+ }
1141
+ document.getElementById('mexport').onclick = async () => {
1142
+ setBackupMsg('exporting...');
1143
+ try {
1144
+ const r = await fetch('/v1/memory/export', {
1145
+ method: 'POST', headers: {'Content-Type': 'application/json'},
1146
+ body: JSON.stringify({session_id: sessionId})});
1147
+ const j = await r.json();
1148
+ if (!r.ok) {
1149
+ setBackupMsg('error: ' + (j.error || r.status), true);
1150
+ return;
1151
+ }
1152
+ const blob = new Blob([JSON.stringify(j.data, null, 2)],
1153
+ {type: 'application/json'});
1154
+ const url = URL.createObjectURL(blob);
1155
+ const a = document.createElement('a');
1156
+ a.href = url;
1157
+ a.download = 'atomzero-' + sessionId + '.json';
1158
+ a.click();
1159
+ URL.revokeObjectURL(url);
1160
+ setBackupMsg('session exported');
1161
+ } catch (e) { setBackupMsg('error: ' + e, true); }
1162
+ };
1163
+ document.getElementById('mimportbtn').onclick = () => {
1164
+ const file = document.getElementById('mimport').files[0];
1165
+ if (!file) { setBackupMsg('choose a JSON backup first', true); return; }
1166
+ if (!confirm('Replace this session memory with the imported file?')) return;
1167
+ const reader = new FileReader();
1168
+ reader.onload = async () => {
1169
+ try {
1170
+ const data = JSON.parse(reader.result);
1171
+ const r = await fetch('/v1/memory/import', {
1172
+ method: 'POST', headers: {'Content-Type': 'application/json'},
1173
+ body: JSON.stringify({session_id: sessionId, data})});
1174
+ const j = await r.json();
1175
+ if (!r.ok) {
1176
+ setBackupMsg('error: ' + (j.error || r.status), true);
1177
+ return;
1178
+ }
1179
+ setBackupMsg('imported facts ' + j.facts + ', episodes ' +
1180
+ j.episodes + ', tasks ' + j.tasks);
1181
+ setTimeout(() => location.reload(), 500);
1182
+ } catch (e) { setBackupMsg('error: ' + e, true); }
1183
+ };
1184
+ reader.readAsText(file);
1185
+ };
1186
+ function setAdapterMsg(text, isError) {
1187
+ const el = document.getElementById('amsg');
1188
+ el.textContent = text;
1189
+ el.style.color = isError ? '#b00020' : '#666';
1190
+ }
1191
+ function renderAdapters(data, active) {
1192
+ const box = document.getElementById('aq');
1193
+ box.textContent = '';
1194
+ if (!data.length) {
1195
+ const empty = document.createElement('div');
1196
+ empty.className = 'hint';
1197
+ empty.textContent = 'No accepted adapter versions yet.';
1198
+ box.appendChild(empty);
1199
+ return;
1200
+ }
1201
+ data.slice().reverse().forEach(item => {
1202
+ const card = document.createElement('div');
1203
+ card.className = 'qitem';
1204
+ const title = document.createElement('div');
1205
+ title.textContent = (item.version || '?') + ' | ' +
1206
+ (item.examples || 0) + ' examples | ' +
1207
+ (item.path === active ? 'ACTIVE' : 'accepted');
1208
+ card.appendChild(title);
1209
+ const path = document.createElement('div');
1210
+ path.className = 'src';
1211
+ path.textContent = item.path || '';
1212
+ card.appendChild(path);
1213
+ const decision = item.decision || {};
1214
+ const metric = document.createElement('div');
1215
+ metric.className = 'src';
1216
+ metric.textContent =
1217
+ 'target ' + ((decision.target_gain || 0) * 100).toFixed(2) + '% | ' +
1218
+ 'bpb ' + ((decision.clean_bpb_reg || 0) * 100).toFixed(2) + '% | ' +
1219
+ 'assistant ' + ((decision.clean_assistant_reg || 0) * 100).toFixed(2) + '%';
1220
+ card.appendChild(metric);
1221
+ box.appendChild(card);
1222
+ });
1223
+ }
1224
+ async function refreshAdapters() {
1225
+ try {
1226
+ const r = await fetch('/v1/learning/adapters');
1227
+ const j = await r.json();
1228
+ if (!r.ok) {
1229
+ setAdapterMsg('adapter error: ' + (j.error || r.status), true);
1230
+ return;
1231
+ }
1232
+ renderAdapters(j.data || [], j.active || '');
1233
+ setAdapterMsg('active: ' + (j.active || 'none'));
1234
+ } catch (e) {
1235
+ setAdapterMsg('adapter error: ' + e, true);
1236
+ }
1237
+ }
1238
+ document.getElementById('arefresh').onclick = () => refreshAdapters();
1239
+ function setGapMsg(text, isError) {
1240
+ const el = document.getElementById('gmsg');
1241
+ el.textContent = text;
1242
+ el.style.color = isError ? '#b00020' : '#666';
1243
+ }
1244
+ function renderGaps(data, stats) {
1245
+ const box = document.getElementById('gq');
1246
+ box.textContent = '';
1247
+ const summary = document.createElement('div');
1248
+ summary.className = 'hint';
1249
+ summary.textContent =
1250
+ 'gaps: ' + (stats.total || 0) + ' total | pending ' +
1251
+ (stats.pending || 0) + ' | proposed ' + (stats.proposed || 0) +
1252
+ ' | dismissed ' + (stats.dismissed || 0);
1253
+ box.appendChild(summary);
1254
+ if (!data.length) {
1255
+ const empty = document.createElement('div');
1256
+ empty.className = 'hint';
1257
+ empty.textContent = 'No knowledge gaps recorded.';
1258
+ box.appendChild(empty);
1259
+ return;
1260
+ }
1261
+ data.forEach(item => {
1262
+ const card = document.createElement('div');
1263
+ card.className = 'qitem';
1264
+ const title = document.createElement('div');
1265
+ title.textContent = item.id + ' | ' + item.status +
1266
+ ' | seen ' + (item.count || 1);
1267
+ card.appendChild(title);
1268
+ const query = document.createElement('div');
1269
+ query.className = 'src';
1270
+ query.textContent = item.query || '';
1271
+ card.appendChild(query);
1272
+ if (item.research_id) {
1273
+ const research = document.createElement('div');
1274
+ research.className = 'src';
1275
+ research.textContent = 'research: ' + item.research_id;
1276
+ card.appendChild(research);
1277
+ }
1278
+ const actions = document.createElement('div');
1279
+ actions.className = 'actions';
1280
+ if (item.status === 'pending') {
1281
+ actions.appendChild(qbutton('Propose research',
1282
+ () => gapAction(item.id, 'propose')));
1283
+ }
1284
+ if (item.status === 'pending' || item.status === 'proposed') {
1285
+ actions.appendChild(qbutton('Dismiss',
1286
+ () => gapAction(item.id, 'dismiss')));
1287
+ }
1288
+ if (actions.childNodes.length) card.appendChild(actions);
1289
+ box.appendChild(card);
1290
+ });
1291
+ }
1292
+ async function refreshGaps() {
1293
+ try {
1294
+ const r = await fetch('/v1/learning/gaps');
1295
+ const j = await r.json();
1296
+ if (!r.ok) {
1297
+ setGapMsg('gap error: ' + (j.error || r.status), true);
1298
+ return;
1299
+ }
1300
+ renderGaps(j.data || [], j.stats || {});
1301
+ } catch (e) {
1302
+ setGapMsg('gap error: ' + e, true);
1303
+ }
1304
+ }
1305
+ async function gapAction(id, action) {
1306
+ setGapMsg(action + ' ' + id + '...');
1307
+ try {
1308
+ const r = await fetch('/v1/learning/gaps/' + action, {
1309
+ method: 'POST', headers: {'Content-Type': 'application/json'},
1310
+ body: JSON.stringify({id})});
1311
+ const j = await r.json();
1312
+ if (!r.ok) {
1313
+ setGapMsg('error: ' + (j.error || r.status), true);
1314
+ return;
1315
+ }
1316
+ setGapMsg(j.research
1317
+ ? (id + ' -> research ' + j.research.id + ' (pending approval)')
1318
+ : (id + ' -> ' + j.status));
1319
+ await refreshGaps();
1320
+ await refreshQueue();
1321
+ } catch (e) {
1322
+ setGapMsg('error: ' + e, true);
1323
+ }
1324
+ }
1325
+ document.getElementById('grefresh').onclick = () => refreshGaps();
1326
+ async function proposeCuriosity() {
1327
+ const button = document.getElementById('gcuriosity');
1328
+ const label = button.textContent;
1329
+ button.disabled = true;
1330
+ button.textContent = 'Proposing...';
1331
+ setGapMsg('proposing pending gaps as curiosity topics...');
1332
+ try {
1333
+ const r = await fetch('/v1/curiosity/propose', {
1334
+ method: 'POST', headers: {'Content-Type': 'application/json'},
1335
+ body: JSON.stringify({limit: 3, pages: 1})});
1336
+ const j = await r.json();
1337
+ if (!r.ok) {
1338
+ setGapMsg('error: ' + (j.error || r.status), true);
1339
+ } else {
1340
+ const count = (j.proposed || []).length;
1341
+ setGapMsg(count + ' curiosity topic(s) pending approval' +
1342
+ ((j.skipped || []).length ? ' | ' + j.skipped.length + ' skipped' : ''));
1343
+ await refreshGaps();
1344
+ await refreshQueue();
1345
+ }
1346
+ } catch (e) {
1347
+ setGapMsg('error: ' + e, true);
1348
+ } finally {
1349
+ button.disabled = false;
1350
+ button.textContent = label;
1351
+ }
1352
+ }
1353
+ document.getElementById('gcuriosity').onclick = proposeCuriosity;
1354
+ function refreshHealth() {
1355
+ fetch('/health').then(r => r.json()).then(j => {
1356
+ const docs = j.documents ? j.documents.chunks : 0;
1357
+ document.getElementById('m').textContent =
1358
+ 'Atom Zero | model: ' + j.model + ' | params: ' + j.params + ' | step: ' + j.step +
1359
+ ' | adapter: ' + (j.adapter ? 'yes' : 'none') + ' | doc chunks: ' + docs +
1360
+ ' | web research: ' + (j.web_research && j.web_research.url_fetch ? 'URL on' : 'Wikipedia') +
1361
+ ' | safety: ' + (j.safety && j.safety.enabled ? 'on' : 'off') +
1362
+ ' | reranker: ' + (j.reranker || 'hashed') +
1363
+ ' | chat: ' + (j.chat_provider && j.chat_provider.enabled ? j.chat_provider.model : 'tiny-50m') +
1364
+ ' | teacher: ' + (j.teacher && j.teacher.enabled ? j.teacher.model : 'off');
1365
+ const teacher = j.teacher || {enabled: false};
1366
+ document.getElementById('task').disabled = !teacher.enabled;
1367
+ document.getElementById('tstatus').textContent = teacher.enabled
1368
+ ? ('Provider: ' + (teacher.label || 'openai-compatible') + ' / ' +
1369
+ teacher.model + ' @ ' + teacher.host)
1370
+ : 'No teacher provider configured; use manual paste.';
1371
+ }).catch(() => { document.getElementById('m').textContent = 'server not ready'; });
1372
+ }
1373
+ restoreChat();
1374
+ refreshHealth();
1375
+ refreshQueue();
1376
+ refreshTeacherNotes();
1377
+ refreshLearning();
1378
+ refreshGaps();
1379
+ refreshDocs();
1380
+ refreshAdapters();
1381
+ </script>"""
1382
+
1383
+
1384
+ def doc_answer(text: str, docs) -> dict | None:
1385
+ """Return grounded notes with sources, or None if nothing matches."""
1386
+ if docs is None:
1387
+ return None
1388
+ hits = docs.search(text, k=3)
1389
+ if not hits:
1390
+ return None
1391
+ notes = []
1392
+ source_map = {}
1393
+ unverified = False
1394
+ for hit in hits[:3]:
1395
+ teacher = hit.get("kind") == "teacher"
1396
+ verified = bool(hit.get("verified", True))
1397
+ if teacher and not verified:
1398
+ label = "teacher note, unverified"
1399
+ elif teacher:
1400
+ reviewer = str(hit.get("reviewer", "")).strip()
1401
+ label = (f"teacher note, verified by {reviewer}"
1402
+ if reviewer else "teacher note, verified")
1403
+ else:
1404
+ label = "source"
1405
+ location = format_citation(hit)
1406
+ notes.append(f"- {best_snippet(text, hit['chunk'])} "
1407
+ f"({label}: {location})")
1408
+ add_citation_source(source_map, hit)
1409
+ if not verified:
1410
+ unverified = True
1411
+ text = (f"I found these notes:\n" + "\n".join(notes) + "\n\n"
1412
+ "This is a local note, not professional advice.")
1413
+ if unverified:
1414
+ text += "\n\nThis includes an unverified teacher-derived note."
1415
+ return {"text": text, "sources": list(source_map.values())}
1416
+
1417
+
1418
+ def chat_prompt(messages) -> str:
1419
+ """Build the chat format used during training."""
1420
+ parts = []
1421
+ for message in (messages or [])[-8:]:
1422
+ role = message.get("role", "user")
1423
+ content = str(message.get("content", "")).strip()
1424
+ if not content:
1425
+ continue
1426
+ tag = "<|assistant|>" if role == "assistant" else "<|user|>"
1427
+ parts.append(f"{tag} {content}")
1428
+ if not parts or not parts[-1].startswith("<|user|>"):
1429
+ parts.append("<|user|> hello")
1430
+ # Training conversations have no <|end|> between a user turn and the
1431
+ # assistant reply; <|end|> closes the whole conversation.
1432
+ return " ".join(parts) + " <|assistant|>"
1433
+
1434
+
1435
+ class Handler(BaseHTTPRequestHandler):
1436
+ protocol_version = "HTTP/1.1"
1437
+
1438
+ def _send(self, code: int, body: bytes, ctype: str = "application/json"):
1439
+ self.send_response(code)
1440
+ self.send_header("Content-Type", ctype)
1441
+ self.send_header("Content-Length", str(len(body)))
1442
+ self.send_header("Access-Control-Allow-Origin", "*")
1443
+ self.end_headers()
1444
+ self.wfile.write(body)
1445
+
1446
+ def _json(self, code: int, payload: dict):
1447
+ self._send(code, json.dumps(payload).encode("utf-8"))
1448
+
1449
+ def do_GET(self):
1450
+ if self.path == "/health":
1451
+ self._json(200, {
1452
+ "ok": True,
1453
+ "model": _state["name"],
1454
+ "params": _state["params"],
1455
+ "config": _state["config"],
1456
+ "step": _state.get("step"),
1457
+ "adapter": _state.get("adapter") or None,
1458
+ "documents": (_state["docs"].stats()
1459
+ if _state.get("docs") else None),
1460
+ "learning_queue": _state.get("queue_path"),
1461
+ "web_research": {
1462
+ "wikipedia_lang": _state.get("wiki_lang"),
1463
+ "url_fetch": bool(_state.get("allow_url_fetch")),
1464
+ },
1465
+ "research_queue": (
1466
+ _state["research_queue"].stats()
1467
+ if _state.get("research_queue") else None),
1468
+ "teacher": (
1469
+ {"enabled": True,
1470
+ "model": _state["teacher"].model,
1471
+ "host": _state["teacher"].host,
1472
+ "label": _state["teacher"].label}
1473
+ if _state.get("teacher") else {"enabled": False}),
1474
+ "chat_provider": (
1475
+ {"enabled": True,
1476
+ "model": _state["chat_provider"].model,
1477
+ "host": _state["chat_provider"].host,
1478
+ "label": _state["chat_provider"].label,
1479
+ "general_chat": bool(_state.get("allow_general_chat"))}
1480
+ if _state.get("chat_provider")
1481
+ else {"enabled": False}),
1482
+ "learning": (
1483
+ _state["feedback"].stats()
1484
+ if _state.get("feedback") else None),
1485
+ "knowledge_gaps": (
1486
+ _state["gaps"].stats()
1487
+ if _state.get("gaps") else None),
1488
+ "safety": (
1489
+ {"enabled": True,
1490
+ "version": _state["safety"].version,
1491
+ **({"ml": True}
1492
+ if getattr(_state["safety"], "ml_enabled", False)
1493
+ else {})}
1494
+ if _state.get("safety") else {"enabled": False}),
1495
+ "reranker": _state.get("reranker", "hashed"),
1496
+ "vector_index": (
1497
+ _state["docs"].vector_index_stats()
1498
+ if _state.get("docs") else None),
1499
+ "sessions": len(_state.get("sessions", {})),
1500
+ })
1501
+ elif self.path == "/v1/models":
1502
+ self._json(200, {"data": [{
1503
+ "id": _state["name"], "object": "model",
1504
+ "owned_by": "streamft",
1505
+ }]})
1506
+ elif self.path == "/metrics":
1507
+ self._json(200, _state["metrics"])
1508
+ elif self.path == "/v1/documents":
1509
+ docs = _state.get("docs")
1510
+ self._json(200, {"data": docs.sources() if docs else []})
1511
+ elif (self.path == "/v1/documents/evidence"
1512
+ or self.path.startswith("/v1/documents/evidence?")):
1513
+ docs = _state.get("docs")
1514
+ if docs is None:
1515
+ return self._json(400, {
1516
+ "error": "no document memory configured"})
1517
+ params = urllib.parse.parse_qs(
1518
+ urllib.parse.urlparse(self.path).query)
1519
+ source = params.get("source", [""])[0]
1520
+ section = params.get("section", [""])[0]
1521
+ try:
1522
+ raw_page = params.get("page", [None])[0]
1523
+ page = (int(raw_page)
1524
+ if raw_page not in (None, "") else None)
1525
+ result = docs.source_evidence(
1526
+ source, page=page, section=section,
1527
+ limit=int(params.get("limit", ["20"])[0]))
1528
+ except SourceNotFound as exc:
1529
+ return self._json(404, {"error": str(exc)})
1530
+ except (TypeError, ValueError) as exc:
1531
+ return self._json(400, {"error": str(exc)})
1532
+ return self._json(200, result)
1533
+ elif self.path == "/v1/teacher/notes":
1534
+ docs = _state.get("docs")
1535
+ notes = ([source for source in docs.sources()
1536
+ if source.get("kind") == "teacher"]
1537
+ if docs else [])
1538
+ self._json(200, {
1539
+ "data": notes,
1540
+ "stats": {
1541
+ "total": len(notes),
1542
+ "verified": sum(1 for note in notes
1543
+ if note.get("verified")),
1544
+ "unverified": sum(1 for note in notes
1545
+ if not note.get("verified")),
1546
+ },
1547
+ })
1548
+ elif self.path == "/v1/learning/queue":
1549
+ store = _state.get("feedback")
1550
+ if store is None:
1551
+ return self._json(400, {
1552
+ "error": "feedback store is not configured"})
1553
+ self._json(200, {"data": store.list(), "stats": store.stats()})
1554
+ elif self.path == "/v1/learning/gaps":
1555
+ store = _state.get("gaps")
1556
+ if store is None:
1557
+ return self._json(400, {
1558
+ "error": "knowledge-gap store is not configured"})
1559
+ self._json(200, {"data": store.list(), "stats": store.stats()})
1560
+ elif self.path == "/v1/learning/adapters":
1561
+ entries = []
1562
+ history = Path(_state.get(
1563
+ "adapter_history_path", "data/adapter_history.jsonl"))
1564
+ if history.exists():
1565
+ for line in history.read_text(encoding="utf-8").splitlines():
1566
+ if line.strip():
1567
+ try:
1568
+ entries.append(json.loads(line))
1569
+ except json.JSONDecodeError:
1570
+ pass
1571
+ active = ""
1572
+ active_path = Path(_state.get(
1573
+ "active_adapter_file", "data/active_adapter.json"))
1574
+ if active_path.exists():
1575
+ try:
1576
+ active = str(json.loads(
1577
+ active_path.read_text(encoding="utf-8")).get("path", ""))
1578
+ except (OSError, json.JSONDecodeError):
1579
+ active = ""
1580
+ self._json(200, {
1581
+ "data": entries, "active": active,
1582
+ "history": str(history), "active_file": str(active_path),
1583
+ })
1584
+ elif self.path == "/v1/curiosity":
1585
+ queue = _state.get("research_queue")
1586
+ gaps = _state.get("gaps")
1587
+ if queue is None or gaps is None:
1588
+ return self._json(400, {
1589
+ "error": "research queue or gap store is not configured"})
1590
+ self._json(200, {
1591
+ "items": curiosity_items(queue),
1592
+ "pending_gaps": sum(
1593
+ 1 for gap in gaps.list()
1594
+ if gap.get("status") == "pending"),
1595
+ "stats": queue.stats(),
1596
+ })
1597
+ elif self.path == "/v1/research/queue":
1598
+ queue = _state.get("research_queue")
1599
+ if queue is None:
1600
+ return self._json(400, {
1601
+ "error": "research queue is not configured"})
1602
+ self._json(200, {"data": self._queue_items(queue),
1603
+ "stats": queue.stats()})
1604
+ elif self.path in ("/", "/index.html"):
1605
+ self._send(200, PAGE.encode("utf-8"), "text/html; charset=utf-8")
1606
+ else:
1607
+ self._json(404, {"error": "not found"})
1608
+
1609
+ def do_POST(self):
1610
+ parsed = urllib.parse.urlparse(self.path)
1611
+ route = parsed.path
1612
+ if route not in ("/generate", "/v1/chat/completions",
1613
+ "/v1/documents", "/v1/documents/search",
1614
+ "/v1/documents/upload", "/v1/documents/forget",
1615
+ "/v1/memory/forget", "/v1/memory/history",
1616
+ "/v1/memory/export", "/v1/memory/import",
1617
+ "/v1/teacher/notes",
1618
+ "/v1/teacher/notes/verify", "/v1/teacher/ask",
1619
+ "/v1/teach", "/v1/learning/feedback",
1620
+ "/v1/learning/queue/approve",
1621
+ "/v1/learning/queue/reject",
1622
+ "/v1/learning/queue/export",
1623
+ "/v1/learning/gaps/propose",
1624
+ "/v1/learning/gaps/dismiss",
1625
+ "/v1/curiosity/propose",
1626
+ "/v1/learn_url", "/v1/research",
1627
+ "/v1/research/queue",
1628
+ "/v1/research/queue/approve",
1629
+ "/v1/research/queue/reject",
1630
+ "/v1/research/queue/run",
1631
+ "/v1/agent/chat", "/v1/agent/plan"):
1632
+ return self._json(404, {"error": "not found"})
1633
+ key = _state.get("api_key") or ""
1634
+ if key:
1635
+ auth = self.headers.get("Authorization", "")
1636
+ if auth != f"Bearer {key}":
1637
+ return self._json(401, {"error": "unauthorized"})
1638
+ try:
1639
+ n = int(self.headers.get("Content-Length", "0"))
1640
+ except ValueError:
1641
+ return self._json(400, {"error": "invalid Content-Length"})
1642
+ if n < 0:
1643
+ return self._json(400, {"error": "invalid Content-Length"})
1644
+ # Raw uploads have their own limit; a 3 MB PDF must not be rejected by
1645
+ # the smaller JSON body cap before _upload sees it.
1646
+ if route == "/v1/documents/upload":
1647
+ return self._upload(parsed.query, n)
1648
+ if n > MAX_BODY_BYTES:
1649
+ self.close_connection = True
1650
+ return self._json(413, {"error": "request body is too large"})
1651
+ try:
1652
+ req = json.loads(self.rfile.read(n) or b"{}")
1653
+ except (json.JSONDecodeError, UnicodeDecodeError):
1654
+ return self._json(400, {"error": "invalid JSON"})
1655
+ if not isinstance(req, dict):
1656
+ return self._json(400, {"error": "JSON object required"})
1657
+ try:
1658
+ if route == "/v1/documents":
1659
+ return self._ingest(req)
1660
+ if route == "/v1/teacher/notes":
1661
+ return self._teacher_note(req)
1662
+ if route == "/v1/teacher/notes/verify":
1663
+ return self._teacher_verify(req)
1664
+ if route == "/v1/teacher/ask":
1665
+ return self._teacher_ask(req)
1666
+ if route == "/v1/learning/feedback":
1667
+ return self._feedback_add(req)
1668
+ if route == "/v1/learning/queue/approve":
1669
+ return self._feedback_update(req, "approve")
1670
+ if route == "/v1/learning/queue/reject":
1671
+ return self._feedback_update(req, "reject")
1672
+ if route == "/v1/learning/queue/export":
1673
+ return self._feedback_export(req)
1674
+ if route == "/v1/learning/gaps/propose":
1675
+ return self._gap_propose(req)
1676
+ if route == "/v1/learning/gaps/dismiss":
1677
+ return self._gap_dismiss(req)
1678
+ if route == "/v1/curiosity/propose":
1679
+ return self._curiosity_propose(req)
1680
+ if route == "/v1/research/queue":
1681
+ return self._queue_add(req)
1682
+ if route == "/v1/research/queue/approve":
1683
+ return self._queue_update(req, "approve")
1684
+ if route == "/v1/research/queue/reject":
1685
+ return self._queue_update(req, "reject")
1686
+ if route == "/v1/research/queue/run":
1687
+ return self._queue_run(req)
1688
+ if route == "/v1/agent/plan":
1689
+ return self._agent_plan(req)
1690
+ if route == "/v1/documents/forget":
1691
+ return self._documents_forget(req)
1692
+ if route == "/v1/memory/forget":
1693
+ return self._memory_forget(req)
1694
+ if route == "/v1/memory/history":
1695
+ return self._memory_history(req)
1696
+ if route == "/v1/memory/export":
1697
+ return self._memory_export(req)
1698
+ if route == "/v1/memory/import":
1699
+ return self._memory_import(req)
1700
+ if self.path == "/v1/documents/search":
1701
+ docs = _state.get("docs")
1702
+ query = str(req.get("query", ""))
1703
+ k = int(req.get("k", 5))
1704
+ hits = docs.search(query, k=k) if docs else []
1705
+ return self._json(200, {"data": hits})
1706
+ if self.path == "/v1/teach":
1707
+ return self._teach(req)
1708
+ if self.path == "/v1/learn_url":
1709
+ if not _state.get("allow_url_fetch"):
1710
+ return self._json(403, {
1711
+ "error": "URL fetch disabled; start with "
1712
+ "--allow-url-fetch"})
1713
+ docs = _state.get("docs")
1714
+ if docs is None:
1715
+ return self._json(400, {"error": "no document memory"})
1716
+ from atomzero.web_research import index_url
1717
+ try:
1718
+ page = index_url(docs, str(req.get("url", "")))
1719
+ except Exception as exc:
1720
+ return self._json(400, {"error": str(exc)})
1721
+ return self._json(200, page)
1722
+ if self.path == "/v1/research":
1723
+ docs = _state.get("docs")
1724
+ if docs is None:
1725
+ return self._json(400, {"error": "no document memory"})
1726
+ from atomzero.web_research import wikipedia_research
1727
+ query = str(req.get("query", "")).strip()
1728
+ if not query:
1729
+ return self._json(400, {"error": "empty query"})
1730
+ try:
1731
+ pages = wikipedia_research(
1732
+ docs, query, _state.get("wiki_lang", "en"),
1733
+ int(req.get("limit", 2)))
1734
+ except Exception as exc:
1735
+ return self._json(400, {"error": str(exc)})
1736
+ return self._json(200, {"pages": pages})
1737
+ session_id = str(req.get("session_id", "")).strip()
1738
+ is_agent = self.path == "/v1/agent/chat"
1739
+ if is_agent or (self.path == "/v1/chat/completions"
1740
+ and session_id):
1741
+ messages = req.get("messages") or []
1742
+ message = str(req.get("message", "")).strip()
1743
+ if not message:
1744
+ message = next(
1745
+ (str(m.get("content", "")) for m in reversed(messages)
1746
+ if m.get("role") == "user"), "")
1747
+ if not message:
1748
+ return self._json(400, {"error": "empty message"})
1749
+ try:
1750
+ session = self._session(session_id or "default")
1751
+ result = session.chat(message)
1752
+ except Exception as exc:
1753
+ return self._json(400, {"error": str(exc)})
1754
+ if is_agent:
1755
+ return self._json(200, {"session_id": session.id,
1756
+ **result})
1757
+ return self._json(200, {
1758
+ "id": "tiny-agent-" + str(abs(hash(result["reply"]))
1759
+ % 10**12),
1760
+ "object": "chat.completion",
1761
+ "model": _state["name"],
1762
+ "choices": [{
1763
+ "index": 0,
1764
+ "message": {"role": "assistant",
1765
+ "content": result["reply"]},
1766
+ "finish_reason": "stop",
1767
+ }],
1768
+ "tool_calls": result.get("tool_calls", []),
1769
+ "sources": result.get("sources", []),
1770
+ })
1771
+ if self.path == "/v1/chat/completions":
1772
+ messages = req.get("messages") or []
1773
+ prompt = chat_prompt(messages)
1774
+ max_new = int(req.get("max_tokens", 64))
1775
+ doc_query = next(
1776
+ (str(m.get("content", "")) for m in reversed(messages)
1777
+ if m.get("role") == "user"), "")
1778
+ else:
1779
+ doc_query = str(req.get("prompt", "hello"))
1780
+ prompt = f"<|user|> {doc_query} <|assistant|>"
1781
+ max_new = int(req.get("max_new_tokens", 64))
1782
+ max_new = max(1, min(max_new, 256))
1783
+ safety = _state.get("safety")
1784
+ verdict = safety.check(doc_query) if safety else None
1785
+ if verdict:
1786
+ grounded = {"text": safety.refusal(verdict["category"]),
1787
+ "sources": []}
1788
+ else:
1789
+ grounded = doc_answer(doc_query, _state.get("docs"))
1790
+ if grounded and safety:
1791
+ safe_text, out_verdict = safety.filter_reply(
1792
+ grounded["text"])
1793
+ if out_verdict:
1794
+ grounded = {**grounded, "text": safe_text,
1795
+ "sources": []}
1796
+ verdict = out_verdict
1797
+ if grounded:
1798
+ _state["metrics"]["requests"] += 1
1799
+ if self.path == "/generate":
1800
+ payload = {
1801
+ "text": grounded["text"],
1802
+ "sources": grounded["sources"],
1803
+ "model": _state["name"],
1804
+ }
1805
+ if verdict:
1806
+ payload["safety"] = verdict
1807
+ return self._json(200, payload)
1808
+ payload = {
1809
+ "id": "tiny-doc-" + str(abs(hash(grounded["text"]))
1810
+ % 10**12),
1811
+ "object": "chat.completion",
1812
+ "model": _state["name"],
1813
+ "sources": grounded["sources"],
1814
+ "choices": [{
1815
+ "index": 0,
1816
+ "message": {"role": "assistant",
1817
+ "content": grounded["text"]},
1818
+ "finish_reason": "stop",
1819
+ }],
1820
+ }
1821
+ if verdict:
1822
+ payload["safety"] = verdict
1823
+ return self._json(200, payload)
1824
+ temperature = float(req.get("temperature", 0.7))
1825
+ top_k = int(req.get("top_k", 20))
1826
+ repetition_penalty = float(req.get("repetition_penalty", 1.1))
1827
+ with _lock:
1828
+ text, stop = generate(
1829
+ _state["model"], _state["tok"], prompt,
1830
+ max_new=max_new, temperature=temperature, top_k=top_k,
1831
+ repetition_penalty=repetition_penalty,
1832
+ banned_ids=(_state["user_id"], _state["assistant_id"]),
1833
+ device=_state["device"])
1834
+ _state["metrics"]["requests"] += 1
1835
+ _state["metrics"]["generated_tokens"] += max_new
1836
+ if safety:
1837
+ safe_text, out_verdict = safety.filter_reply(text)
1838
+ if out_verdict:
1839
+ text = safe_text
1840
+ verdict = out_verdict
1841
+ if self.path == "/generate":
1842
+ payload = {"text": text, "stop": stop,
1843
+ "model": _state["name"]}
1844
+ if verdict:
1845
+ payload["safety"] = verdict
1846
+ return self._json(200, payload)
1847
+ payload = {
1848
+ "id": "tiny-" + str(abs(hash(text)) % 10**12),
1849
+ "object": "chat.completion",
1850
+ "model": _state["name"],
1851
+ "choices": [{
1852
+ "index": 0,
1853
+ "message": {"role": "assistant", "content": text},
1854
+ "finish_reason": "stop",
1855
+ }],
1856
+ }
1857
+ if verdict:
1858
+ payload["safety"] = verdict
1859
+ return self._json(200, payload)
1860
+ except Exception as exc: # pragma: no cover
1861
+ return self._json(400, {"error": str(exc)})
1862
+
1863
+ def _feedback_store(self):
1864
+ store = _state.get("feedback")
1865
+ if store is None:
1866
+ raise FeedbackError("feedback store is not configured")
1867
+ return store
1868
+
1869
+ def _feedback_add(self, req: dict):
1870
+ try:
1871
+ item = self._feedback_store().add(
1872
+ req.get("user", ""), req.get("assistant", ""),
1873
+ req.get("rating", 0), req.get("corrected", ""),
1874
+ req.get("session_id", ""))
1875
+ except FeedbackError as exc:
1876
+ return self._json(400, {"error": str(exc)})
1877
+ return self._json(200, item)
1878
+
1879
+ def _feedback_update(self, req: dict, action: str):
1880
+ try:
1881
+ store = self._feedback_store()
1882
+ item_id = str(req.get("id", ""))
1883
+ item = (store.approve(item_id) if action == "approve"
1884
+ else store.reject(item_id))
1885
+ except FeedbackNotFound as exc:
1886
+ return self._json(404, {"error": str(exc)})
1887
+ except FeedbackError as exc:
1888
+ return self._json(400, {"error": str(exc)})
1889
+ return self._json(200, item)
1890
+
1891
+ def _feedback_export(self, req: dict):
1892
+ try:
1893
+ result = self._feedback_store().export_training(
1894
+ _state.get("feedback_export_path",
1895
+ "data/feedback_training.jsonl"))
1896
+ except FeedbackError as exc:
1897
+ return self._json(400, {"error": str(exc)})
1898
+ return self._json(200, result)
1899
+
1900
+ def _gap_store(self):
1901
+ store = _state.get("gaps")
1902
+ if store is None:
1903
+ raise KnowledgeGapError(
1904
+ "knowledge-gap store is not configured")
1905
+ return store
1906
+
1907
+ def _gap_propose(self, req: dict):
1908
+ try:
1909
+ gaps = self._gap_store()
1910
+ gap_id = str(req.get("id", ""))
1911
+ gap = gaps.get(gap_id)
1912
+ if gap.get("status") != "pending":
1913
+ raise KnowledgeGapError(
1914
+ f"gap is already {gap.get('status')}")
1915
+ item = self._queue().add(
1916
+ gap["query"], _state.get("wiki_lang", "en"),
1917
+ int(req.get("pages", 1)))
1918
+ gap = gaps.mark_proposed(gap_id, item["id"])
1919
+ except GapNotFound as exc:
1920
+ return self._json(404, {"error": str(exc)})
1921
+ except (KnowledgeGapError, ResearchQueueError, ValueError) as exc:
1922
+ return self._json(400, {"error": str(exc)})
1923
+ return self._json(200, {
1924
+ "gap": gap, "research": item,
1925
+ "next": (f"approve {item['id']} in the research queue, then run "
1926
+ "the scheduled worker"),
1927
+ })
1928
+
1929
+ def _curiosity_propose(self, req: dict):
1930
+ try:
1931
+ result = propose_from_gaps(
1932
+ self._gap_store(), self._queue(),
1933
+ limit=req.get("limit", 3),
1934
+ lang=_state.get("wiki_lang", "en"),
1935
+ pages=int(req.get("pages", 1)),
1936
+ min_count=req.get("min_count", 1))
1937
+ except (CuriosityError, KnowledgeGapError, ResearchQueueError,
1938
+ ValueError) as exc:
1939
+ return self._json(400, {"error": str(exc)})
1940
+ return self._json(200, result)
1941
+
1942
+ def _gap_dismiss(self, req: dict):
1943
+ try:
1944
+ item = self._gap_store().dismiss(str(req.get("id", "")))
1945
+ except GapNotFound as exc:
1946
+ return self._json(404, {"error": str(exc)})
1947
+ except KnowledgeGapError as exc:
1948
+ return self._json(400, {"error": str(exc)})
1949
+ return self._json(200, item)
1950
+
1951
+ def _queue(self):
1952
+ queue = _state.get("research_queue")
1953
+ if queue is None:
1954
+ raise ResearchQueueError("research queue is not configured")
1955
+ return queue
1956
+
1957
+ def _queue_items(self, queue) -> list[dict]:
1958
+ items = queue.list()
1959
+ docs = _state.get("docs")
1960
+ if docs is None:
1961
+ return items
1962
+ notes = {source["source"]: source for source in docs.sources()
1963
+ if source.get("kind") == "teacher"}
1964
+ for item in items:
1965
+ teacher = item.get("teacher")
1966
+ if not isinstance(teacher, dict):
1967
+ continue
1968
+ note = notes.get(teacher.get("source", ""))
1969
+ if note is not None:
1970
+ teacher["verified"] = bool(note.get("verified"))
1971
+ teacher["reviewer"] = note.get("reviewer", "")
1972
+ teacher["verified_at"] = note.get("verified_at", "")
1973
+ return items
1974
+
1975
+ def _queue_add(self, req: dict):
1976
+ try:
1977
+ item = self._queue().add(
1978
+ req.get("topic", ""), req.get("lang", "en"),
1979
+ req.get("pages", 2))
1980
+ except ResearchQueueError as exc:
1981
+ return self._json(400, {"error": str(exc)})
1982
+ return self._json(200, item)
1983
+
1984
+ def _queue_update(self, req: dict, action: str):
1985
+ try:
1986
+ queue = self._queue()
1987
+ item_id = str(req.get("id", ""))
1988
+ item = (queue.approve(item_id) if action == "approve"
1989
+ else queue.reject(item_id))
1990
+ except ItemNotFound as exc:
1991
+ return self._json(404, {"error": str(exc)})
1992
+ except ResearchQueueError as exc:
1993
+ return self._json(400, {"error": str(exc)})
1994
+ return self._json(200, item)
1995
+
1996
+ def _queue_run(self, req: dict):
1997
+ docs = _state.get("docs")
1998
+ if docs is None:
1999
+ return self._json(400, {"error": "no document memory configured"})
2000
+ try:
2001
+ queue = self._queue()
2002
+ from atomzero.web_research import wikipedia_research
2003
+ ran = queue.run_approved(
2004
+ docs, req.get("limit", 1), wikipedia_research,
2005
+ _state.get("teacher"))
2006
+ except ResearchQueueError as exc:
2007
+ return self._json(400, {"error": str(exc)})
2008
+ return self._json(200, {"ran": ran, "stats": queue.stats()})
2009
+
2010
+ def _teacher_verify(self, req: dict):
2011
+ docs = _state.get("docs")
2012
+ if docs is None:
2013
+ return self._json(400, {"error": "no document memory configured"})
2014
+ try:
2015
+ note = docs.verify_source(
2016
+ req.get("source", ""), req.get("reviewer", ""))
2017
+ except SourceNotFound as exc:
2018
+ return self._json(404, {"error": str(exc)})
2019
+ except ValueError as exc:
2020
+ return self._json(400, {"error": str(exc)})
2021
+ return self._json(200, {"ok": True, **note})
2022
+
2023
+ def _teacher_ask(self, req: dict):
2024
+ provider = _state.get("teacher")
2025
+ if provider is None:
2026
+ return self._json(400, {
2027
+ "error": "teacher provider is not configured"})
2028
+ prompt = str(req.get("prompt") or "").strip()
2029
+ topic = str(req.get("topic") or "").strip()
2030
+ if not prompt and topic:
2031
+ prompt = f"Give a concise, factual summary of: {topic}"
2032
+ if not prompt:
2033
+ return self._json(400, {"error": "prompt or topic is required"})
2034
+ if len(prompt) > 20000:
2035
+ return self._json(400, {"error": "prompt is too long"})
2036
+ try:
2037
+ answer = provider.ask(
2038
+ prompt,
2039
+ system=("You are a concise research assistant. "
2040
+ "If you are unsure, say so."),
2041
+ max_tokens=req.get("max_tokens", 512))
2042
+ except TeacherProviderError as exc:
2043
+ return self._json(502, {"error": str(exc)})
2044
+ try:
2045
+ result = add_teacher_note(
2046
+ _state.get("docs"), req.get("teacher") or provider.label,
2047
+ provider.model, topic or "Provider answer", answer,
2048
+ req.get("source_url", ""))
2049
+ except TeacherNoteTooLarge as exc:
2050
+ return self._json(413, {"error": str(exc)})
2051
+ except ValueError as exc:
2052
+ return self._json(400, {"error": str(exc)})
2053
+ metrics = _state.get("metrics")
2054
+ if isinstance(metrics, dict):
2055
+ metrics["teacher_notes"] = metrics.get("teacher_notes", 0) + 1
2056
+ return self._json(200, result)
2057
+
2058
+ def _teacher_note(self, req: dict):
2059
+ try:
2060
+ result = add_teacher_note(
2061
+ _state.get("docs"), req.get("teacher", "manual"),
2062
+ req.get("model", ""),
2063
+ req.get("topic") or req.get("question", ""),
2064
+ req.get("answer", ""), req.get("source_url", ""))
2065
+ except TeacherNoteTooLarge as exc:
2066
+ return self._json(413, {"error": str(exc)})
2067
+ except ValueError as exc:
2068
+ return self._json(400, {"error": str(exc)})
2069
+ metrics = _state.get("metrics")
2070
+ if isinstance(metrics, dict):
2071
+ metrics["teacher_notes"] = metrics.get("teacher_notes", 0) + 1
2072
+ return self._json(200, result)
2073
+
2074
+ def _ingest(self, req: dict):
2075
+ try:
2076
+ result = ingest_document(
2077
+ _state.get("docs"),
2078
+ req.get("source") or req.get("filename"),
2079
+ req.get("text") or req.get("content") or "",
2080
+ req.get("title", ""))
2081
+ except DocumentTooLarge as exc:
2082
+ return self._json(413, {"error": str(exc)})
2083
+ except ValueError as exc:
2084
+ return self._json(400, {"error": str(exc)})
2085
+ metrics = _state.get("metrics")
2086
+ if isinstance(metrics, dict):
2087
+ metrics["documents"] = metrics.get("documents", 0) + 1
2088
+ return self._json(200, result)
2089
+
2090
+ def _upload(self, query: str, n: int):
2091
+ params = urllib.parse.parse_qs(query)
2092
+ source = (params.get("source", [""])[0]
2093
+ or params.get("filename", [""])[0])
2094
+ title = params.get("title", [""])[0]
2095
+ if n > MAX_DOC_BYTES:
2096
+ self.close_connection = True
2097
+ return self._json(413, {
2098
+ "error": f"document is too large (max {MAX_DOC_BYTES} bytes)"})
2099
+ raw = self.rfile.read(n)
2100
+ try:
2101
+ result = ingest_upload(
2102
+ _state.get("docs"), source, raw, title)
2103
+ except DocumentTooLarge as exc:
2104
+ return self._json(413, {"error": str(exc)})
2105
+ except ValueError as exc:
2106
+ return self._json(400, {"error": str(exc)})
2107
+ metrics = _state.get("metrics")
2108
+ if isinstance(metrics, dict):
2109
+ metrics["documents"] = metrics.get("documents", 0) + 1
2110
+ return self._json(200, result)
2111
+
2112
+ def _documents_forget(self, req: dict):
2113
+ docs = _state.get("docs")
2114
+ if docs is None:
2115
+ return self._json(400, {"error": "no document memory configured"})
2116
+ try:
2117
+ result = docs.forget(str(req.get("source", "")))
2118
+ except SourceNotFound as exc:
2119
+ return self._json(404, {"error": str(exc)})
2120
+ except ValueError as exc:
2121
+ return self._json(400, {"error": str(exc)})
2122
+ return self._json(200, result)
2123
+
2124
+ def _memory_forget(self, req: dict):
2125
+ session_id = str(req.get("session_id", "")).strip()
2126
+ if not session_id:
2127
+ return self._json(400, {"error": "session_id is required"})
2128
+ text = str(req.get("text", "")).strip()
2129
+ clear_all = bool(req.get("all"))
2130
+ if not text and not clear_all:
2131
+ return self._json(400, {"error": "text or all is required"})
2132
+ try:
2133
+ session = self._session(session_id)
2134
+ result = session.forget("" if clear_all else text)
2135
+ except Exception as exc:
2136
+ return self._json(400, {"error": str(exc)})
2137
+ return self._json(200, {"session_id": session_id, **result})
2138
+
2139
+ def _memory_export(self, req: dict):
2140
+ session_id = str(req.get("session_id", "")).strip()
2141
+ if not session_id:
2142
+ return self._json(400, {"error": "session_id is required"})
2143
+ try:
2144
+ session = self._session(session_id)
2145
+ data = session.export_memory()
2146
+ except Exception as exc:
2147
+ return self._json(400, {"error": str(exc)})
2148
+ return self._json(200, {"session_id": session_id, "data": data})
2149
+
2150
+ def _memory_import(self, req: dict):
2151
+ session_id = str(req.get("session_id", "")).strip()
2152
+ data = req.get("data")
2153
+ if not session_id:
2154
+ return self._json(400, {"error": "session_id is required"})
2155
+ if not isinstance(data, dict):
2156
+ return self._json(400, {"error": "data must be an object"})
2157
+ if len(json.dumps(data)) > 1_000_000:
2158
+ return self._json(413, {"error": "memory import is too large"})
2159
+ try:
2160
+ session = self._session(session_id)
2161
+ result = session.import_memory(data)
2162
+ except (OSError, ValueError) as exc:
2163
+ return self._json(400, {"error": str(exc)})
2164
+ return self._json(200, {"session_id": session_id, **result})
2165
+
2166
+ def _memory_history(self, req: dict):
2167
+ session_id = str(req.get("session_id", "")).strip()
2168
+ if not session_id:
2169
+ return self._json(400, {"error": "session_id is required"})
2170
+ try:
2171
+ limit = int(req.get("limit", 50))
2172
+ except (TypeError, ValueError):
2173
+ return self._json(400, {"error": "limit must be an integer"})
2174
+ limit = max(1, min(limit, 200))
2175
+ try:
2176
+ session = self._session(session_id)
2177
+ episodes = session.human.data.get("episodes", [])
2178
+ except Exception as exc:
2179
+ return self._json(400, {"error": str(exc)})
2180
+ return self._json(200, {
2181
+ "session_id": session_id,
2182
+ "data": episodes[-limit:],
2183
+ })
2184
+
2185
+ def _session(self, session_id: str):
2186
+ from atomzero.agent_runtime import AgentSession
2187
+ sessions = _state.setdefault("sessions", {})
2188
+ if session_id not in sessions:
2189
+ folder = Path(_state.get("session_dir", "data/sessions"))
2190
+ folder.mkdir(parents=True, exist_ok=True)
2191
+ sessions[session_id] = AgentSession(
2192
+ session_id, folder / f"{session_id}.json",
2193
+ model=_state.get("model"), tokenizer=_state.get("tok"),
2194
+ device=_state.get("device"), docs=_state.get("docs"),
2195
+ wiki_lang=_state.get("wiki_lang", "en"),
2196
+ gap_store=_state.get("gaps"),
2197
+ safety=_state.get("safety"),
2198
+ generator=_state.get("generator"),
2199
+ allow_general_chat=_state.get("allow_general_chat", False))
2200
+ return sessions[session_id]
2201
+
2202
+ def _agent_plan(self, req: dict):
2203
+ session_id = str(req.get("session_id", "")).strip()
2204
+ if not session_id:
2205
+ return self._json(400, {"error": "session_id is required"})
2206
+ confirm = req.get("confirm", False)
2207
+ if not isinstance(confirm, bool):
2208
+ return self._json(400, {"error": "confirm must be a boolean"})
2209
+ try:
2210
+ session = self._session(session_id)
2211
+ result = execute_plan(
2212
+ session, req.get("steps"), confirm=confirm)
2213
+ except ToolPlanError as exc:
2214
+ return self._json(400, {"error": str(exc)})
2215
+ except Exception as exc:
2216
+ return self._json(400, {"error": str(exc)})
2217
+ return self._json(200, result)
2218
+
2219
+ def _teach(self, req: dict):
2220
+ fact = str(req.get("fact", "")).strip()
2221
+ user = str(req.get("user", "")).strip()
2222
+ assistant = str(req.get("assistant", "")).strip()
2223
+ if not fact and not (user and assistant):
2224
+ return self._json(400, {"error": "provide fact or user+assistant"})
2225
+ if len(fact) > 4000 or len(user) > 2000 or len(assistant) > 2000:
2226
+ return self._json(400, {"error": "input too long"})
2227
+ result = {"ok": True}
2228
+ if fact:
2229
+ docs = _state.get("docs")
2230
+ if docs is None:
2231
+ return self._json(400, {"error": "no document memory configured"})
2232
+ result["fact_chunks"] = docs.add_text(
2233
+ "user_teach", fact, title="Taught facts", replace=False)
2234
+ if user and assistant:
2235
+ queue = Path(_state.get("queue_path",
2236
+ "data/learning_queue.jsonl"))
2237
+ queue.parent.mkdir(parents=True, exist_ok=True)
2238
+ with queue.open("a", encoding="utf-8") as handle:
2239
+ handle.write(json.dumps({
2240
+ "user": user, "assistant": assistant,
2241
+ "ts": int(time.time()),
2242
+ }, ensure_ascii=False) + "\n")
2243
+ result["skill_queued"] = True
2244
+ return self._json(200, result)
2245
+
2246
+ def log_message(self, fmt, *args):
2247
+ return
2248
+
2249
+
2250
+ def _attach_reranker(docs, mode: str = "auto",
2251
+ neural_model: str = "models/all-MiniLM-L6-v2") -> str:
2252
+ """Attach the best available reranker; never fail server startup."""
2253
+ if docs is None or mode == "hashed":
2254
+ return "hashed"
2255
+ try:
2256
+ neural = NeuralReranker(neural_model)
2257
+ neural.embed(["warmup"])
2258
+ docs.reranker = neural
2259
+ return "neural"
2260
+ except (NeuralEmbedderError, OSError, RuntimeError) as exc:
2261
+ level = "warning" if mode == "neural" else "note"
2262
+ print(f"{level}: neural reranker disabled, using hashed "
2263
+ f"embeddings: {exc}", flush=True)
2264
+ return "hashed"
2265
+
2266
+
2267
+ def serve(model_path: str, tokenizer_path: str, host: str = "127.0.0.1",
2268
+ port: int = 8013, api_key: str = "", adapter: str = "",
2269
+ docs_path: str = "",
2270
+ queue_path: str = "data/learning_queue.jsonl",
2271
+ allow_url_fetch: bool = False, wiki_lang: str = "en",
2272
+ session_dir: str = "data/sessions",
2273
+ research_queue_path: str = "data/research_queue.json",
2274
+ feedback_path: str = "data/feedback_queue.json",
2275
+ feedback_export_path: str = "data/feedback_training.jsonl",
2276
+ teacher_url: str = "", teacher_model: str = "",
2277
+ teacher_timeout: int = 60,
2278
+ teacher_label: str = "openai-compatible",
2279
+ active_adapter_file: str = "data/active_adapter.json",
2280
+ gap_store_path: str = "data/knowledge_gaps.json",
2281
+ safety_filter: bool = False,
2282
+ reranker: str = "auto",
2283
+ neural_model: str = "models/all-MiniLM-L6-v2",
2284
+ prewarm_vectors: bool = True,
2285
+ vector_index: str = "auto",
2286
+ safety_ml: bool = True,
2287
+ chat_provider_url: str = "",
2288
+ chat_provider_model: str = "",
2289
+ chat_provider_api_key: str = "",
2290
+ chat_provider_timeout: int = 60,
2291
+ chat_provider_label: str = "openai-compatible",
2292
+ allow_general_chat: bool = False) -> int:
2293
+ adapter = resolve_adapter(adapter, active_adapter_file)
2294
+ root = Path(model_path)
2295
+ ck = torch.load(root / "model.pt", map_location="cpu", weights_only=False)
2296
+ cfg = ck["config"]
2297
+ model = TransformerLM(cfg["vocab"], cfg["dim"], cfg["layers"],
2298
+ cfg["heads"], cfg["ff"])
2299
+ model.load_state_dict(ck["state"])
2300
+ if adapter:
2301
+ from atomzero.lora import apply_adapter
2302
+ apply_adapter(model, adapter)
2303
+ device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
2304
+ model.to(device)
2305
+ model.eval()
2306
+ from tokenizers import Tokenizer
2307
+ tok = Tokenizer.from_file(tokenizer_path)
2308
+ docs = None
2309
+ if docs_path:
2310
+ from atomzero.doc_memory import DocMemory
2311
+ docs = DocMemory(docs_path)
2312
+ docs.vector_index_mode = vector_index
2313
+ active_reranker = _attach_reranker(docs, reranker, neural_model)
2314
+ safety = None
2315
+ if safety_filter:
2316
+ from atomzero.ml_safety import build_safety_filter
2317
+ encoder = getattr(docs, "reranker", None) if docs else None
2318
+ safety = build_safety_filter(prefer_ml=safety_ml, encoder=encoder)
2319
+ if docs is not None and prewarm_vectors:
2320
+ try:
2321
+ stats = docs.prewarm_vectors()
2322
+ print(f"vector index: {stats['chunks']} chunks, "
2323
+ f"dim {stats['dim']}, {stats['backend']}, "
2324
+ f"{stats['build_s']:.3f}s", flush=True)
2325
+ except (OSError, RuntimeError, ValueError) as exc:
2326
+ print(f"warning: vector prewarm failed: {exc}", flush=True)
2327
+ research_queue = None
2328
+ if research_queue_path:
2329
+ try:
2330
+ research_queue = ResearchQueue(research_queue_path)
2331
+ except ResearchQueueError as exc:
2332
+ print(f"warning: research queue disabled: {exc}", flush=True)
2333
+ feedback = None
2334
+ if feedback_path:
2335
+ try:
2336
+ feedback = FeedbackStore(feedback_path)
2337
+ except FeedbackError as exc:
2338
+ print(f"warning: feedback store disabled: {exc}", flush=True)
2339
+ gaps = None
2340
+ if gap_store_path:
2341
+ try:
2342
+ gaps = KnowledgeGapStore(gap_store_path)
2343
+ except KnowledgeGapError as exc:
2344
+ print(f"warning: knowledge-gap store disabled: {exc}", flush=True)
2345
+ teacher = None
2346
+ if teacher_url or teacher_model:
2347
+ try:
2348
+ teacher = TeacherProvider(
2349
+ teacher_url, teacher_model,
2350
+ os.environ.get("TEACHER_API_KEY", ""),
2351
+ teacher_timeout, teacher_label)
2352
+ except TeacherProviderError as exc:
2353
+ print(f"warning: teacher provider disabled: {exc}", flush=True)
2354
+ chat_provider = None
2355
+ if chat_provider_url or chat_provider_model:
2356
+ try:
2357
+ chat_provider = TeacherProvider(
2358
+ chat_provider_url, chat_provider_model,
2359
+ chat_provider_api_key or os.environ.get(
2360
+ "CHAT_PROVIDER_API_KEY", ""),
2361
+ chat_provider_timeout, chat_provider_label)
2362
+ print(f"chat provider: {chat_provider.label} / "
2363
+ f"{chat_provider.model} @ {chat_provider.host}",
2364
+ flush=True)
2365
+ except TeacherProviderError as exc:
2366
+ print(f"warning: chat provider disabled: {exc}", flush=True)
2367
+
2368
+ def chat_generate(prompt: str):
2369
+ if chat_provider is None:
2370
+ return None
2371
+ try:
2372
+ return chat_provider.ask(
2373
+ prompt,
2374
+ system=("You are a concise, direct local assistant. Answer "
2375
+ "the user's request without refusing benign topics."),
2376
+ max_tokens=512)
2377
+ except TeacherProviderError as exc:
2378
+ print(f"warning: chat provider request failed: {exc}", flush=True)
2379
+ return None
2380
+
2381
+ _state.update({
2382
+ "model": model,
2383
+ "tok": tok,
2384
+ "docs": docs,
2385
+ "research_queue": research_queue,
2386
+ "research_queue_path": research_queue_path,
2387
+ "feedback": feedback,
2388
+ "feedback_path": feedback_path,
2389
+ "feedback_export_path": feedback_export_path,
2390
+ "gaps": gaps,
2391
+ "gap_store_path": gap_store_path,
2392
+ "adapter_history_path": "data/adapter_history.jsonl",
2393
+ "active_adapter_file": active_adapter_file,
2394
+ "safety": safety,
2395
+ "reranker": active_reranker,
2396
+ "teacher": teacher,
2397
+ "chat_provider": chat_provider,
2398
+ "generator": chat_generate if chat_provider else None,
2399
+ "allow_general_chat": bool(allow_general_chat),
2400
+ "adapter": adapter,
2401
+ "queue_path": queue_path,
2402
+ "allow_url_fetch": allow_url_fetch,
2403
+ "wiki_lang": wiki_lang,
2404
+ "session_dir": session_dir,
2405
+ "sessions": {},
2406
+ "device": device,
2407
+ "name": root.name,
2408
+ "params": sum(p.numel() for p in model.parameters()),
2409
+ "config": cfg,
2410
+ "step": ck.get("step"),
2411
+ "api_key": api_key or os.environ.get("CHAT_API_KEY", ""),
2412
+ "user_id": tok.token_to_id("<|user|>"),
2413
+ "assistant_id": tok.token_to_id("<|assistant|>"),
2414
+ "metrics": {"requests": 0, "generated_tokens": 0, "documents": 0,
2415
+ "teacher_notes": 0},
2416
+ })
2417
+ httpd = ThreadingHTTPServer((host, port), Handler)
2418
+ print(f"Atom Zero serving {model_path} at http://{host}:{port}",
2419
+ flush=True)
2420
+ try:
2421
+ httpd.serve_forever()
2422
+ except KeyboardInterrupt:
2423
+ pass
2424
+ finally:
2425
+ httpd.server_close()
2426
+ return 0
2427
+
2428
+
2429
+ def main():
2430
+ ap = argparse.ArgumentParser()
2431
+ ap.add_argument("--model",
2432
+ default="checkpoints/transformer_behavior_50m_scaled")
2433
+ ap.add_argument("--tokenizer", default="data/chat_en_bpe8k.json")
2434
+ ap.add_argument("--host", default="127.0.0.1")
2435
+ ap.add_argument("--port", type=int, default=8013)
2436
+ ap.add_argument("--api-key", default="")
2437
+ ap.add_argument("--adapter", default="")
2438
+ ap.add_argument("--docs", default="")
2439
+ ap.add_argument("--queue", default="data/learning_queue.jsonl")
2440
+ ap.add_argument("--allow-url-fetch", action="store_true")
2441
+ ap.add_argument("--wiki-lang", default="en")
2442
+ ap.add_argument("--session-dir", default="data/sessions")
2443
+ ap.add_argument("--research-queue", default="data/research_queue.json")
2444
+ ap.add_argument("--feedback-store", default="data/feedback_queue.json")
2445
+ ap.add_argument("--feedback-export", default="data/feedback_training.jsonl")
2446
+ ap.add_argument("--gap-store", default="data/knowledge_gaps.json")
2447
+ ap.add_argument("--safety-filter", action="store_true")
2448
+ ap.add_argument("--no-safety-ml", dest="safety_ml",
2449
+ action="store_false", default=True,
2450
+ help="use only the exact rule filter, not the local ML head")
2451
+ ap.add_argument("--reranker", choices=("auto", "hashed", "neural"),
2452
+ default="auto",
2453
+ help="auto uses the local neural model when present, "
2454
+ "otherwise hashed n-gram embeddings")
2455
+ ap.add_argument("--neural-model", default="models/all-MiniLM-L6-v2")
2456
+ ap.add_argument("--no-prewarm", dest="prewarm_vectors",
2457
+ action="store_false", default=True,
2458
+ help="skip building the in-memory vector index at startup")
2459
+ ap.add_argument("--vector-index", choices=("auto", "ann", "exact"),
2460
+ default="auto",
2461
+ help="auto uses IVF ANN above 256 chunks, exact below; "
2462
+ "exact fallback is always available")
2463
+ ap.add_argument("--chat-provider-url", default="",
2464
+ help="optional OpenAI-compatible local chat backend, "
2465
+ "e.g. http://127.0.0.1:11434/v1")
2466
+ ap.add_argument("--chat-provider-model", default="",
2467
+ help="model name for --chat-provider-url, e.g. llama3.2:3b")
2468
+ ap.add_argument("--chat-provider-api-key", default="")
2469
+ ap.add_argument("--chat-provider-timeout", type=int, default=60)
2470
+ ap.add_argument("--chat-provider-label", default="local-provider")
2471
+ ap.add_argument("--allow-general-chat", action="store_true",
2472
+ help="let the provider answer when no local source matches")
2473
+ ap.add_argument("--teacher-url", default="")
2474
+ ap.add_argument("--teacher-model", default="")
2475
+ ap.add_argument("--teacher-timeout", type=int, default=60)
2476
+ ap.add_argument("--teacher-label", default="openai-compatible")
2477
+ ap.add_argument("--active-adapter-file", default="data/active_adapter.json")
2478
+ args = ap.parse_args()
2479
+ return serve(args.model, args.tokenizer, args.host, args.port,
2480
+ args.api_key, args.adapter, args.docs, args.queue,
2481
+ args.allow_url_fetch, args.wiki_lang, args.session_dir,
2482
+ args.research_queue, args.feedback_store,
2483
+ args.feedback_export, args.teacher_url, args.teacher_model,
2484
+ args.teacher_timeout, args.teacher_label,
2485
+ args.active_adapter_file, args.gap_store,
2486
+ args.safety_filter, args.reranker, args.neural_model,
2487
+ args.prewarm_vectors, args.vector_index, args.safety_ml,
2488
+ args.chat_provider_url, args.chat_provider_model,
2489
+ args.chat_provider_api_key, args.chat_provider_timeout,
2490
+ args.chat_provider_label, args.allow_general_chat)
2491
+
2492
+
2493
+ if __name__ == "__main__":
2494
+ raise SystemExit(main())