atomzero 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
atomzero/curiosity.py ADDED
@@ -0,0 +1,66 @@
1
+ """Curiosity proposals: turn unanswered questions into reviewed topics.
2
+
3
+ The assistant does not browse or ask a teacher on its own. It records a
4
+ knowledge gap when no local source matches. This module proposes those gaps as
5
+ **pending** research topics; a human must approve them before the worker fetches
6
+ sources or calls the teacher. Nothing here changes model weights.
7
+ """
8
+ from __future__ import annotations
9
+
10
+ from atomzero.knowledge_gaps import GapNotFound, KnowledgeGapError
11
+ from atomzero.research_queue import ResearchQueueError
12
+
13
+
14
+ class CuriosityError(ValueError):
15
+ pass
16
+
17
+
18
+ def propose_from_gaps(gaps, queue, limit: int = 3, lang: str = "en",
19
+ pages: int = 1, min_count: int = 1) -> dict:
20
+ """Create pending research items from pending knowledge gaps.
21
+
22
+ The operation is deliberately local: no network, teacher, or weight update.
23
+ """
24
+ if gaps is None or queue is None:
25
+ raise CuriosityError("knowledge gaps and research queue are required")
26
+ try:
27
+ limit = max(1, min(int(limit), 10))
28
+ min_count = max(1, int(min_count))
29
+ except (TypeError, ValueError) as exc:
30
+ raise CuriosityError("limit/min_count must be integers") from exc
31
+ pending = [gap for gap in gaps.list()
32
+ if gap.get("status") == "pending"
33
+ and int(gap.get("count", 1)) >= min_count]
34
+ pending.sort(key=lambda gap: (-int(gap.get("count", 1)),
35
+ str(gap.get("first_seen", ""))))
36
+ proposed = []
37
+ skipped = []
38
+ for gap in pending:
39
+ if len(proposed) >= limit:
40
+ break
41
+ try:
42
+ item = queue.add(gap["query"], lang, pages,
43
+ kind="curiosity", origin_id=gap["id"])
44
+ gap = gaps.mark_proposed(gap["id"], item["id"])
45
+ except (KnowledgeGapError, GapNotFound, ResearchQueueError,
46
+ ValueError) as exc:
47
+ skipped.append({"gap": gap.get("id", ""),
48
+ "query": gap.get("query", ""),
49
+ "error": str(exc)[:200]})
50
+ continue
51
+ proposed.append({"gap": gap, "research": item})
52
+ return {
53
+ "proposed": proposed,
54
+ "skipped": skipped,
55
+ "next": ("approve the pending research items, then run the worker; "
56
+ "approval is required before any fetch or teacher call"),
57
+ }
58
+
59
+
60
+ def curiosity_items(queue, limit: int = 20) -> list[dict]:
61
+ """Return research items created by the curiosity path."""
62
+ if queue is None:
63
+ return []
64
+ items = [item for item in queue.list()
65
+ if item.get("kind") == "curiosity"]
66
+ return items[:max(1, int(limit))]
atomzero/doc_ingest.py ADDED
@@ -0,0 +1,312 @@
1
+ """Extract text blocks with page/section provenance from local documents.
2
+
3
+ No network and no OCR. The extractor keeps the source structure that the
4
+ retrieval layer needs for citations:
5
+
6
+ * PDF: one block per page (1-based ``page``)
7
+ * DOCX/Markdown/HTML: blocks grouped under their current heading
8
+ (``section``)
9
+
10
+ Binary formats are optional at import time so the rest of the package keeps
11
+ working when only the text path is installed.
12
+ """
13
+ from __future__ import annotations
14
+
15
+ import io
16
+ import re
17
+ from html.parser import HTMLParser
18
+ from pathlib import Path
19
+
20
+ TEXT_SUFFIXES = {".txt", ".md", ".markdown", ".html", ".htm"}
21
+ SUPPORTED_SUFFIXES = TEXT_SUFFIXES | {".pdf", ".docx"}
22
+
23
+
24
+ class DocumentExtractError(ValueError):
25
+ """Raised when a document cannot be converted into usable text."""
26
+
27
+
28
+ def _clean(text: str) -> str:
29
+ text = str(text or "").replace("\r\n", "\n").replace("\r", "\n")
30
+ text = text.replace("\x00", " ")
31
+ text = re.sub(r"[ \t]+\n", "\n", text)
32
+ text = re.sub(r"\n{3,}", "\n\n", text)
33
+ return text.strip()
34
+
35
+
36
+ def _block(text: str, page: int | None = None,
37
+ section: str = "") -> dict | None:
38
+ text = _clean(text)
39
+ if not text:
40
+ return None
41
+ return {
42
+ "text": text,
43
+ "page": page,
44
+ "section": _clean(section).replace("\n", " "),
45
+ }
46
+
47
+
48
+ def _markdown_blocks(text: str) -> list[dict]:
49
+ blocks: list[dict] = []
50
+ section = ""
51
+ current: list[str] = []
52
+ in_fence = False
53
+
54
+ def flush() -> None:
55
+ nonlocal current
56
+ block = _block("\n".join(current), section=section)
57
+ if block:
58
+ blocks.append(block)
59
+ current = []
60
+
61
+ for line in _clean(text).splitlines():
62
+ stripped = line.strip()
63
+ if stripped.startswith("```"):
64
+ in_fence = not in_fence
65
+ current.append(line)
66
+ continue
67
+ heading = (None if in_fence
68
+ else re.match(r"^(#{1,6})\s+(.+?)\s*#*\s*$", line))
69
+ if heading:
70
+ flush()
71
+ section = heading.group(2).strip()
72
+ continue
73
+ current.append(line)
74
+ flush()
75
+ if not blocks:
76
+ block = _block(text)
77
+ if block:
78
+ blocks.append(block)
79
+ return blocks
80
+
81
+
82
+ class _HTMLBlocks(HTMLParser):
83
+ """Small block-level HTML reader; no third-party parser required."""
84
+
85
+ _BLOCKS = {"p", "li", "div", "section", "article", "tr", "td", "th"}
86
+ _HEADINGS = {"h1", "h2", "h3", "h4", "h5", "h6"}
87
+
88
+ def __init__(self) -> None:
89
+ super().__init__(convert_charrefs=True)
90
+ self.title = ""
91
+ self.blocks: list[dict] = []
92
+ self._skip = 0
93
+ self._in_title = False
94
+ self._heading_tag = ""
95
+ self._heading_parts: list[str] = []
96
+ self._section = ""
97
+ self._buffer: list[str] = []
98
+
99
+ def _flush(self) -> None:
100
+ text = " ".join(" ".join(self._buffer).split())
101
+ if text:
102
+ self.blocks.append({
103
+ "text": text, "page": None, "section": self._section})
104
+ self._buffer = []
105
+
106
+ def handle_starttag(self, tag, attrs):
107
+ tag = tag.lower()
108
+ if tag in ("script", "style", "noscript", "svg"):
109
+ self._skip += 1
110
+ elif tag == "title":
111
+ self._in_title = True
112
+ elif tag in self._HEADINGS:
113
+ self._flush()
114
+ self._heading_tag = tag
115
+ self._heading_parts = []
116
+ elif tag in self._BLOCKS or tag == "br":
117
+ self._flush()
118
+
119
+ def handle_endtag(self, tag):
120
+ tag = tag.lower()
121
+ if tag in ("script", "style", "noscript", "svg"):
122
+ if self._skip:
123
+ self._skip -= 1
124
+ elif tag == "title":
125
+ self._in_title = False
126
+ elif tag == self._heading_tag:
127
+ heading = " ".join("".join(self._heading_parts).split())
128
+ if heading:
129
+ self._section = heading
130
+ self._heading_tag = ""
131
+ self._heading_parts = []
132
+ self._flush()
133
+ elif tag in self._BLOCKS:
134
+ self._flush()
135
+
136
+ def handle_data(self, data):
137
+ if self._in_title:
138
+ self.title += data
139
+ elif self._skip:
140
+ return
141
+ elif self._heading_tag:
142
+ self._heading_parts.append(data)
143
+ else:
144
+ self._buffer.append(data)
145
+
146
+ def finish(self) -> tuple[str, list[dict]]:
147
+ self._flush()
148
+ return self.title.strip(), self.blocks
149
+
150
+
151
+ def _html_blocks(html: str) -> tuple[str, list[dict]]:
152
+ parser = _HTMLBlocks()
153
+ parser.feed(html)
154
+ parser.close()
155
+ return parser.finish()
156
+
157
+
158
+ def _pdf_blocks(data: bytes) -> tuple[str, list[dict]]:
159
+ try:
160
+ from pypdf import PdfReader
161
+ except ImportError as exc: # pragma: no cover - declared dependency
162
+ raise DocumentExtractError(
163
+ "pypdf is required for PDF ingestion") from exc
164
+ try:
165
+ reader = PdfReader(io.BytesIO(data))
166
+ except Exception as exc:
167
+ raise DocumentExtractError(f"cannot read PDF: {exc}") from exc
168
+ if getattr(reader, "is_encrypted", False):
169
+ try:
170
+ reader.decrypt("")
171
+ except Exception as exc:
172
+ raise DocumentExtractError(
173
+ "PDF is encrypted and cannot be read") from exc
174
+ blocks: list[dict] = []
175
+ for number, page in enumerate(reader.pages, start=1):
176
+ try:
177
+ text = page.extract_text() or ""
178
+ except Exception:
179
+ text = ""
180
+ block = _block(text, page=number)
181
+ if block:
182
+ blocks.append(block)
183
+ if not blocks:
184
+ raise DocumentExtractError(
185
+ "PDF has no extractable text; scanned PDFs need OCR")
186
+ title = ""
187
+ try:
188
+ metadata = reader.metadata
189
+ if metadata is not None:
190
+ title = str(metadata.title or "")
191
+ except Exception:
192
+ title = ""
193
+ return title, blocks
194
+
195
+
196
+ def _docx_blocks(data: bytes) -> tuple[str, list[dict]]:
197
+ try:
198
+ from docx import Document
199
+ except ImportError as exc: # pragma: no cover - declared dependency
200
+ raise DocumentExtractError(
201
+ "python-docx is required for DOCX ingestion") from exc
202
+ try:
203
+ document = Document(io.BytesIO(data))
204
+ except Exception as exc:
205
+ raise DocumentExtractError(f"cannot read DOCX: {exc}") from exc
206
+ from docx.oxml.ns import qn
207
+ from docx.table import Table
208
+ from docx.text.paragraph import Paragraph
209
+
210
+ blocks: list[dict] = []
211
+ section = ""
212
+ current: list[str] = []
213
+
214
+ def flush() -> None:
215
+ nonlocal current
216
+ block = _block("\n\n".join(current), section=section)
217
+ if block:
218
+ blocks.append(block)
219
+ current = []
220
+
221
+ def iter_blocks():
222
+ body = document.element.body
223
+ for child in body.iterchildren():
224
+ if child.tag == qn("w:p"):
225
+ yield Paragraph(child, document)
226
+ elif child.tag == qn("w:tbl"):
227
+ yield Table(child, document)
228
+
229
+ for item in iter_blocks():
230
+ if isinstance(item, Paragraph):
231
+ text = (item.text or "").strip()
232
+ if not text:
233
+ continue
234
+ style = ""
235
+ try:
236
+ style = (item.style.name or "").lower()
237
+ except Exception:
238
+ style = ""
239
+ if style.startswith("heading"):
240
+ flush()
241
+ section = text
242
+ continue
243
+ current.append(text)
244
+ if sum(len(part) for part in current) > 1800:
245
+ flush()
246
+ continue
247
+
248
+ # A table keeps its position relative to the paragraphs around it.
249
+ flush()
250
+ rows = []
251
+ for row in item.rows:
252
+ cells = [(cell.text or "").strip() for cell in row.cells]
253
+ if any(cells):
254
+ rows.append(" | ".join(cells))
255
+ block = _block("\n".join(rows), section=section or "Table")
256
+ if block:
257
+ blocks.append(block)
258
+ flush()
259
+
260
+ if not blocks:
261
+ raise DocumentExtractError("DOCX has no extractable text")
262
+ title = ""
263
+ try:
264
+ title = str(document.core_properties.title or "")
265
+ except Exception:
266
+ title = ""
267
+ return title, blocks
268
+
269
+
270
+ def extract_bytes(filename: str, data: bytes,
271
+ title: str = "") -> dict:
272
+ """Convert file bytes into ``title``, ``format`` and provenance blocks."""
273
+ name = Path(str(filename or "document")).name
274
+ suffix = Path(name).suffix.lower()
275
+ fallback = Path(name).stem or "document"
276
+ if suffix == ".pdf":
277
+ meta_title, blocks = _pdf_blocks(data)
278
+ elif suffix == ".docx":
279
+ meta_title, blocks = _docx_blocks(data)
280
+ elif suffix in (".html", ".htm"):
281
+ html = data.decode("utf-8-sig", errors="replace")
282
+ meta_title, blocks = _html_blocks(html)
283
+ elif suffix in (".md", ".markdown"):
284
+ text = data.decode("utf-8-sig", errors="replace")
285
+ meta_title, blocks = "", _markdown_blocks(text)
286
+ elif not suffix or suffix == ".txt":
287
+ # Extension-less uploads keep the old UTF-8 text path.
288
+ text = data.decode("utf-8-sig", errors="replace")
289
+ meta_title, blocks = "", [_block(text)]
290
+ else:
291
+ raise DocumentExtractError(
292
+ f"unsupported document type: {suffix}; "
293
+ "use .txt, .md, .html, .pdf or .docx")
294
+ blocks = [block for block in blocks if block]
295
+ if not blocks:
296
+ raise DocumentExtractError(
297
+ f"document has no extractable text: {name}")
298
+ return {
299
+ "title": str(title or "").strip() or meta_title.strip() or fallback,
300
+ "blocks": blocks,
301
+ "format": suffix.lstrip(".") or "text",
302
+ "source": name,
303
+ }
304
+
305
+
306
+ def extract_file(path: str | Path, title: str = "") -> dict:
307
+ p = Path(path)
308
+ try:
309
+ data = p.read_bytes()
310
+ except OSError as exc:
311
+ raise DocumentExtractError(f"cannot read {p}: {exc}") from exc
312
+ return extract_bytes(p.name, data, title=title)