atomzero 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- atomzero/__init__.py +8 -0
- atomzero/agent_runtime.py +225 -0
- atomzero/chat_human.py +486 -0
- atomzero/chat_server.py +2494 -0
- atomzero/curiosity.py +66 -0
- atomzero/doc_ingest.py +312 -0
- atomzero/doc_memory.py +1112 -0
- atomzero/feedback.py +188 -0
- atomzero/knowledge_gaps.py +157 -0
- atomzero/lora.py +140 -0
- atomzero/ml_safety.py +165 -0
- atomzero/neural_embedder.py +108 -0
- atomzero/research_queue.py +330 -0
- atomzero/safety.py +226 -0
- atomzero/teacher_notes.py +78 -0
- atomzero/teacher_provider.py +104 -0
- atomzero/tool_planner.py +207 -0
- atomzero/transformer.py +102 -0
- atomzero/web_research.py +165 -0
- atomzero-0.1.0.dist-info/METADATA +96 -0
- atomzero-0.1.0.dist-info/RECORD +26 -0
- atomzero-0.1.0.dist-info/WHEEL +5 -0
- atomzero-0.1.0.dist-info/entry_points.txt +2 -0
- atomzero-0.1.0.dist-info/licenses/LICENSE +202 -0
- atomzero-0.1.0.dist-info/licenses/NOTICE +24 -0
- atomzero-0.1.0.dist-info/top_level.txt +1 -0
atomzero/curiosity.py
ADDED
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
"""Curiosity proposals: turn unanswered questions into reviewed topics.
|
|
2
|
+
|
|
3
|
+
The assistant does not browse or ask a teacher on its own. It records a
|
|
4
|
+
knowledge gap when no local source matches. This module proposes those gaps as
|
|
5
|
+
**pending** research topics; a human must approve them before the worker fetches
|
|
6
|
+
sources or calls the teacher. Nothing here changes model weights.
|
|
7
|
+
"""
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from atomzero.knowledge_gaps import GapNotFound, KnowledgeGapError
|
|
11
|
+
from atomzero.research_queue import ResearchQueueError
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class CuriosityError(ValueError):
|
|
15
|
+
pass
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def propose_from_gaps(gaps, queue, limit: int = 3, lang: str = "en",
|
|
19
|
+
pages: int = 1, min_count: int = 1) -> dict:
|
|
20
|
+
"""Create pending research items from pending knowledge gaps.
|
|
21
|
+
|
|
22
|
+
The operation is deliberately local: no network, teacher, or weight update.
|
|
23
|
+
"""
|
|
24
|
+
if gaps is None or queue is None:
|
|
25
|
+
raise CuriosityError("knowledge gaps and research queue are required")
|
|
26
|
+
try:
|
|
27
|
+
limit = max(1, min(int(limit), 10))
|
|
28
|
+
min_count = max(1, int(min_count))
|
|
29
|
+
except (TypeError, ValueError) as exc:
|
|
30
|
+
raise CuriosityError("limit/min_count must be integers") from exc
|
|
31
|
+
pending = [gap for gap in gaps.list()
|
|
32
|
+
if gap.get("status") == "pending"
|
|
33
|
+
and int(gap.get("count", 1)) >= min_count]
|
|
34
|
+
pending.sort(key=lambda gap: (-int(gap.get("count", 1)),
|
|
35
|
+
str(gap.get("first_seen", ""))))
|
|
36
|
+
proposed = []
|
|
37
|
+
skipped = []
|
|
38
|
+
for gap in pending:
|
|
39
|
+
if len(proposed) >= limit:
|
|
40
|
+
break
|
|
41
|
+
try:
|
|
42
|
+
item = queue.add(gap["query"], lang, pages,
|
|
43
|
+
kind="curiosity", origin_id=gap["id"])
|
|
44
|
+
gap = gaps.mark_proposed(gap["id"], item["id"])
|
|
45
|
+
except (KnowledgeGapError, GapNotFound, ResearchQueueError,
|
|
46
|
+
ValueError) as exc:
|
|
47
|
+
skipped.append({"gap": gap.get("id", ""),
|
|
48
|
+
"query": gap.get("query", ""),
|
|
49
|
+
"error": str(exc)[:200]})
|
|
50
|
+
continue
|
|
51
|
+
proposed.append({"gap": gap, "research": item})
|
|
52
|
+
return {
|
|
53
|
+
"proposed": proposed,
|
|
54
|
+
"skipped": skipped,
|
|
55
|
+
"next": ("approve the pending research items, then run the worker; "
|
|
56
|
+
"approval is required before any fetch or teacher call"),
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def curiosity_items(queue, limit: int = 20) -> list[dict]:
|
|
61
|
+
"""Return research items created by the curiosity path."""
|
|
62
|
+
if queue is None:
|
|
63
|
+
return []
|
|
64
|
+
items = [item for item in queue.list()
|
|
65
|
+
if item.get("kind") == "curiosity"]
|
|
66
|
+
return items[:max(1, int(limit))]
|
atomzero/doc_ingest.py
ADDED
|
@@ -0,0 +1,312 @@
|
|
|
1
|
+
"""Extract text blocks with page/section provenance from local documents.
|
|
2
|
+
|
|
3
|
+
No network and no OCR. The extractor keeps the source structure that the
|
|
4
|
+
retrieval layer needs for citations:
|
|
5
|
+
|
|
6
|
+
* PDF: one block per page (1-based ``page``)
|
|
7
|
+
* DOCX/Markdown/HTML: blocks grouped under their current heading
|
|
8
|
+
(``section``)
|
|
9
|
+
|
|
10
|
+
Binary formats are optional at import time so the rest of the package keeps
|
|
11
|
+
working when only the text path is installed.
|
|
12
|
+
"""
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import io
|
|
16
|
+
import re
|
|
17
|
+
from html.parser import HTMLParser
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
|
|
20
|
+
TEXT_SUFFIXES = {".txt", ".md", ".markdown", ".html", ".htm"}
|
|
21
|
+
SUPPORTED_SUFFIXES = TEXT_SUFFIXES | {".pdf", ".docx"}
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class DocumentExtractError(ValueError):
|
|
25
|
+
"""Raised when a document cannot be converted into usable text."""
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _clean(text: str) -> str:
|
|
29
|
+
text = str(text or "").replace("\r\n", "\n").replace("\r", "\n")
|
|
30
|
+
text = text.replace("\x00", " ")
|
|
31
|
+
text = re.sub(r"[ \t]+\n", "\n", text)
|
|
32
|
+
text = re.sub(r"\n{3,}", "\n\n", text)
|
|
33
|
+
return text.strip()
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _block(text: str, page: int | None = None,
|
|
37
|
+
section: str = "") -> dict | None:
|
|
38
|
+
text = _clean(text)
|
|
39
|
+
if not text:
|
|
40
|
+
return None
|
|
41
|
+
return {
|
|
42
|
+
"text": text,
|
|
43
|
+
"page": page,
|
|
44
|
+
"section": _clean(section).replace("\n", " "),
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _markdown_blocks(text: str) -> list[dict]:
|
|
49
|
+
blocks: list[dict] = []
|
|
50
|
+
section = ""
|
|
51
|
+
current: list[str] = []
|
|
52
|
+
in_fence = False
|
|
53
|
+
|
|
54
|
+
def flush() -> None:
|
|
55
|
+
nonlocal current
|
|
56
|
+
block = _block("\n".join(current), section=section)
|
|
57
|
+
if block:
|
|
58
|
+
blocks.append(block)
|
|
59
|
+
current = []
|
|
60
|
+
|
|
61
|
+
for line in _clean(text).splitlines():
|
|
62
|
+
stripped = line.strip()
|
|
63
|
+
if stripped.startswith("```"):
|
|
64
|
+
in_fence = not in_fence
|
|
65
|
+
current.append(line)
|
|
66
|
+
continue
|
|
67
|
+
heading = (None if in_fence
|
|
68
|
+
else re.match(r"^(#{1,6})\s+(.+?)\s*#*\s*$", line))
|
|
69
|
+
if heading:
|
|
70
|
+
flush()
|
|
71
|
+
section = heading.group(2).strip()
|
|
72
|
+
continue
|
|
73
|
+
current.append(line)
|
|
74
|
+
flush()
|
|
75
|
+
if not blocks:
|
|
76
|
+
block = _block(text)
|
|
77
|
+
if block:
|
|
78
|
+
blocks.append(block)
|
|
79
|
+
return blocks
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
class _HTMLBlocks(HTMLParser):
|
|
83
|
+
"""Small block-level HTML reader; no third-party parser required."""
|
|
84
|
+
|
|
85
|
+
_BLOCKS = {"p", "li", "div", "section", "article", "tr", "td", "th"}
|
|
86
|
+
_HEADINGS = {"h1", "h2", "h3", "h4", "h5", "h6"}
|
|
87
|
+
|
|
88
|
+
def __init__(self) -> None:
|
|
89
|
+
super().__init__(convert_charrefs=True)
|
|
90
|
+
self.title = ""
|
|
91
|
+
self.blocks: list[dict] = []
|
|
92
|
+
self._skip = 0
|
|
93
|
+
self._in_title = False
|
|
94
|
+
self._heading_tag = ""
|
|
95
|
+
self._heading_parts: list[str] = []
|
|
96
|
+
self._section = ""
|
|
97
|
+
self._buffer: list[str] = []
|
|
98
|
+
|
|
99
|
+
def _flush(self) -> None:
|
|
100
|
+
text = " ".join(" ".join(self._buffer).split())
|
|
101
|
+
if text:
|
|
102
|
+
self.blocks.append({
|
|
103
|
+
"text": text, "page": None, "section": self._section})
|
|
104
|
+
self._buffer = []
|
|
105
|
+
|
|
106
|
+
def handle_starttag(self, tag, attrs):
|
|
107
|
+
tag = tag.lower()
|
|
108
|
+
if tag in ("script", "style", "noscript", "svg"):
|
|
109
|
+
self._skip += 1
|
|
110
|
+
elif tag == "title":
|
|
111
|
+
self._in_title = True
|
|
112
|
+
elif tag in self._HEADINGS:
|
|
113
|
+
self._flush()
|
|
114
|
+
self._heading_tag = tag
|
|
115
|
+
self._heading_parts = []
|
|
116
|
+
elif tag in self._BLOCKS or tag == "br":
|
|
117
|
+
self._flush()
|
|
118
|
+
|
|
119
|
+
def handle_endtag(self, tag):
|
|
120
|
+
tag = tag.lower()
|
|
121
|
+
if tag in ("script", "style", "noscript", "svg"):
|
|
122
|
+
if self._skip:
|
|
123
|
+
self._skip -= 1
|
|
124
|
+
elif tag == "title":
|
|
125
|
+
self._in_title = False
|
|
126
|
+
elif tag == self._heading_tag:
|
|
127
|
+
heading = " ".join("".join(self._heading_parts).split())
|
|
128
|
+
if heading:
|
|
129
|
+
self._section = heading
|
|
130
|
+
self._heading_tag = ""
|
|
131
|
+
self._heading_parts = []
|
|
132
|
+
self._flush()
|
|
133
|
+
elif tag in self._BLOCKS:
|
|
134
|
+
self._flush()
|
|
135
|
+
|
|
136
|
+
def handle_data(self, data):
|
|
137
|
+
if self._in_title:
|
|
138
|
+
self.title += data
|
|
139
|
+
elif self._skip:
|
|
140
|
+
return
|
|
141
|
+
elif self._heading_tag:
|
|
142
|
+
self._heading_parts.append(data)
|
|
143
|
+
else:
|
|
144
|
+
self._buffer.append(data)
|
|
145
|
+
|
|
146
|
+
def finish(self) -> tuple[str, list[dict]]:
|
|
147
|
+
self._flush()
|
|
148
|
+
return self.title.strip(), self.blocks
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def _html_blocks(html: str) -> tuple[str, list[dict]]:
|
|
152
|
+
parser = _HTMLBlocks()
|
|
153
|
+
parser.feed(html)
|
|
154
|
+
parser.close()
|
|
155
|
+
return parser.finish()
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def _pdf_blocks(data: bytes) -> tuple[str, list[dict]]:
|
|
159
|
+
try:
|
|
160
|
+
from pypdf import PdfReader
|
|
161
|
+
except ImportError as exc: # pragma: no cover - declared dependency
|
|
162
|
+
raise DocumentExtractError(
|
|
163
|
+
"pypdf is required for PDF ingestion") from exc
|
|
164
|
+
try:
|
|
165
|
+
reader = PdfReader(io.BytesIO(data))
|
|
166
|
+
except Exception as exc:
|
|
167
|
+
raise DocumentExtractError(f"cannot read PDF: {exc}") from exc
|
|
168
|
+
if getattr(reader, "is_encrypted", False):
|
|
169
|
+
try:
|
|
170
|
+
reader.decrypt("")
|
|
171
|
+
except Exception as exc:
|
|
172
|
+
raise DocumentExtractError(
|
|
173
|
+
"PDF is encrypted and cannot be read") from exc
|
|
174
|
+
blocks: list[dict] = []
|
|
175
|
+
for number, page in enumerate(reader.pages, start=1):
|
|
176
|
+
try:
|
|
177
|
+
text = page.extract_text() or ""
|
|
178
|
+
except Exception:
|
|
179
|
+
text = ""
|
|
180
|
+
block = _block(text, page=number)
|
|
181
|
+
if block:
|
|
182
|
+
blocks.append(block)
|
|
183
|
+
if not blocks:
|
|
184
|
+
raise DocumentExtractError(
|
|
185
|
+
"PDF has no extractable text; scanned PDFs need OCR")
|
|
186
|
+
title = ""
|
|
187
|
+
try:
|
|
188
|
+
metadata = reader.metadata
|
|
189
|
+
if metadata is not None:
|
|
190
|
+
title = str(metadata.title or "")
|
|
191
|
+
except Exception:
|
|
192
|
+
title = ""
|
|
193
|
+
return title, blocks
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
def _docx_blocks(data: bytes) -> tuple[str, list[dict]]:
|
|
197
|
+
try:
|
|
198
|
+
from docx import Document
|
|
199
|
+
except ImportError as exc: # pragma: no cover - declared dependency
|
|
200
|
+
raise DocumentExtractError(
|
|
201
|
+
"python-docx is required for DOCX ingestion") from exc
|
|
202
|
+
try:
|
|
203
|
+
document = Document(io.BytesIO(data))
|
|
204
|
+
except Exception as exc:
|
|
205
|
+
raise DocumentExtractError(f"cannot read DOCX: {exc}") from exc
|
|
206
|
+
from docx.oxml.ns import qn
|
|
207
|
+
from docx.table import Table
|
|
208
|
+
from docx.text.paragraph import Paragraph
|
|
209
|
+
|
|
210
|
+
blocks: list[dict] = []
|
|
211
|
+
section = ""
|
|
212
|
+
current: list[str] = []
|
|
213
|
+
|
|
214
|
+
def flush() -> None:
|
|
215
|
+
nonlocal current
|
|
216
|
+
block = _block("\n\n".join(current), section=section)
|
|
217
|
+
if block:
|
|
218
|
+
blocks.append(block)
|
|
219
|
+
current = []
|
|
220
|
+
|
|
221
|
+
def iter_blocks():
|
|
222
|
+
body = document.element.body
|
|
223
|
+
for child in body.iterchildren():
|
|
224
|
+
if child.tag == qn("w:p"):
|
|
225
|
+
yield Paragraph(child, document)
|
|
226
|
+
elif child.tag == qn("w:tbl"):
|
|
227
|
+
yield Table(child, document)
|
|
228
|
+
|
|
229
|
+
for item in iter_blocks():
|
|
230
|
+
if isinstance(item, Paragraph):
|
|
231
|
+
text = (item.text or "").strip()
|
|
232
|
+
if not text:
|
|
233
|
+
continue
|
|
234
|
+
style = ""
|
|
235
|
+
try:
|
|
236
|
+
style = (item.style.name or "").lower()
|
|
237
|
+
except Exception:
|
|
238
|
+
style = ""
|
|
239
|
+
if style.startswith("heading"):
|
|
240
|
+
flush()
|
|
241
|
+
section = text
|
|
242
|
+
continue
|
|
243
|
+
current.append(text)
|
|
244
|
+
if sum(len(part) for part in current) > 1800:
|
|
245
|
+
flush()
|
|
246
|
+
continue
|
|
247
|
+
|
|
248
|
+
# A table keeps its position relative to the paragraphs around it.
|
|
249
|
+
flush()
|
|
250
|
+
rows = []
|
|
251
|
+
for row in item.rows:
|
|
252
|
+
cells = [(cell.text or "").strip() for cell in row.cells]
|
|
253
|
+
if any(cells):
|
|
254
|
+
rows.append(" | ".join(cells))
|
|
255
|
+
block = _block("\n".join(rows), section=section or "Table")
|
|
256
|
+
if block:
|
|
257
|
+
blocks.append(block)
|
|
258
|
+
flush()
|
|
259
|
+
|
|
260
|
+
if not blocks:
|
|
261
|
+
raise DocumentExtractError("DOCX has no extractable text")
|
|
262
|
+
title = ""
|
|
263
|
+
try:
|
|
264
|
+
title = str(document.core_properties.title or "")
|
|
265
|
+
except Exception:
|
|
266
|
+
title = ""
|
|
267
|
+
return title, blocks
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
def extract_bytes(filename: str, data: bytes,
|
|
271
|
+
title: str = "") -> dict:
|
|
272
|
+
"""Convert file bytes into ``title``, ``format`` and provenance blocks."""
|
|
273
|
+
name = Path(str(filename or "document")).name
|
|
274
|
+
suffix = Path(name).suffix.lower()
|
|
275
|
+
fallback = Path(name).stem or "document"
|
|
276
|
+
if suffix == ".pdf":
|
|
277
|
+
meta_title, blocks = _pdf_blocks(data)
|
|
278
|
+
elif suffix == ".docx":
|
|
279
|
+
meta_title, blocks = _docx_blocks(data)
|
|
280
|
+
elif suffix in (".html", ".htm"):
|
|
281
|
+
html = data.decode("utf-8-sig", errors="replace")
|
|
282
|
+
meta_title, blocks = _html_blocks(html)
|
|
283
|
+
elif suffix in (".md", ".markdown"):
|
|
284
|
+
text = data.decode("utf-8-sig", errors="replace")
|
|
285
|
+
meta_title, blocks = "", _markdown_blocks(text)
|
|
286
|
+
elif not suffix or suffix == ".txt":
|
|
287
|
+
# Extension-less uploads keep the old UTF-8 text path.
|
|
288
|
+
text = data.decode("utf-8-sig", errors="replace")
|
|
289
|
+
meta_title, blocks = "", [_block(text)]
|
|
290
|
+
else:
|
|
291
|
+
raise DocumentExtractError(
|
|
292
|
+
f"unsupported document type: {suffix}; "
|
|
293
|
+
"use .txt, .md, .html, .pdf or .docx")
|
|
294
|
+
blocks = [block for block in blocks if block]
|
|
295
|
+
if not blocks:
|
|
296
|
+
raise DocumentExtractError(
|
|
297
|
+
f"document has no extractable text: {name}")
|
|
298
|
+
return {
|
|
299
|
+
"title": str(title or "").strip() or meta_title.strip() or fallback,
|
|
300
|
+
"blocks": blocks,
|
|
301
|
+
"format": suffix.lstrip(".") or "text",
|
|
302
|
+
"source": name,
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
def extract_file(path: str | Path, title: str = "") -> dict:
|
|
307
|
+
p = Path(path)
|
|
308
|
+
try:
|
|
309
|
+
data = p.read_bytes()
|
|
310
|
+
except OSError as exc:
|
|
311
|
+
raise DocumentExtractError(f"cannot read {p}: {exc}") from exc
|
|
312
|
+
return extract_bytes(p.name, data, title=title)
|