brainiac-cli 0.16.0__py3-none-any.whl

brain/ingest/__init__.py

@@ -0,0 +1,19 @@
+"""ING-01/ING-02 — document ingestion pipeline (ADR-0003 Ruling 1).
+
+Drop a file into ``<vault>/inbox/``; the HOST-only ``brain ingest`` verb (also
+folded into every host ``brain sync``, per the s01 cadence amendment) extracts
+it to Markdown, archives the untouched original immutably under
+``raw/originals/``, and commits the extracted source through the existing
+audited ``write_note`` path. Unhandled/failed files quarantine to
+``inbox/_quarantine/<reason>/`` — never silently dropped.
+
+Public surface: :func:`run_ingest` (orchestrator) and :func:`capability_report`
+(which handlers are usable given installed deps) — both re-exported from
+``.pipeline`` and ``.handlers``.
+"""
+from __future__ import annotations
+
+from .handlers import ExtractResult, Handler, capability_report
+from .pipeline import inbox_dir, run_ingest
+
+__all__ = ["ExtractResult", "Handler", "capability_report", "inbox_dir", "run_ingest"]

brain/ingest/handlers/__init__.py

@@ -0,0 +1,43 @@
+"""Extension -> Handler registry (ING-01)."""
+from __future__ import annotations
+
+from .base import ExtractResult, Handler, density_gate, strip_control_chars
+from .docx import DocxHandler
+from .email import EmailHandler
+from .html import HtmlHandler
+from .image import ImageHandler
+from .pdf import PdfHandler
+from .pptx import PptxHandler
+from .text import TextHandler
+from .xlsx import XlsxHandler
+from .zip import ZipHandler
+
+ALL_HANDLERS: tuple[type[Handler], ...] = (
+    PdfHandler, DocxHandler, PptxHandler, XlsxHandler, TextHandler,
+    ImageHandler, EmailHandler, HtmlHandler, ZipHandler,
+)
+
+REGISTRY: dict[str, type[Handler]] = {
+    ext: handler for handler in ALL_HANDLERS for ext in handler.extensions
+}
+
+
+def handler_for(path) -> type[Handler] | None:
+    from pathlib import Path
+
+    return REGISTRY.get(Path(path).suffix.lower())
+
+
+def capability_report() -> dict[str, dict]:
+    """Which extension handlers are available right now (ING-01 dep probe)."""
+    return {
+        ext: {"handler": handler.__name__, "dependency": handler.dependency_name,
+              "available": handler.available()}
+        for ext, handler in REGISTRY.items()
+    }
+
+
+__all__ = [
+    "ExtractResult", "Handler", "density_gate", "strip_control_chars",
+    "ALL_HANDLERS", "REGISTRY", "handler_for", "capability_report",
+]

brain/ingest/handlers/base.py

@@ -0,0 +1,95 @@
+"""Handler contract (ING-01/ING-02, ADR-0003 Ruling 1).
+
+Every format handler implements one function contract: given a Path, return an
+``ExtractResult``. Handlers never touch the vault, the index, or the audit
+chain — that is the orchestrator's job (``brain.ingest.run_ingest``). This
+keeps a handler pure and trivially testable: bytes in, Markdown (or a
+quarantine reason) out.
+"""
+from __future__ import annotations
+
+import re
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+# Below this many non-whitespace characters, a "successfully extracted"
+# document is treated as empty/near-empty content — the OHRBench finding that
+# upstream extraction failure (not the write path) is the dominant corpus-
+# corruption vector. Quarantine, never sign a near-empty source.
+MIN_CONTENT_CHARS = 40
+
+
+@dataclass
+class ExtractResult:
+    """Outcome of one handler's extraction attempt.
+
+    ``quarantine_reason`` is ``None`` on success. ``markdown``/``warnings``/
+    ``metadata`` are always populated (possibly empty) so callers never branch
+    on attribute presence.
+    """
+
+    markdown: str = ""
+    warnings: list[str] = field(default_factory=list)
+    quarantine_reason: str | None = None
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+    @property
+    def ok(self) -> bool:
+        return self.quarantine_reason is None
+
+    @staticmethod
+    def quarantine(reason: str, *, warnings: list[str] | None = None) -> "ExtractResult":
+        return ExtractResult(quarantine_reason=reason, warnings=list(warnings or []))
+
+
+def density_gate(markdown: str, *, min_chars: int = MIN_CONTENT_CHARS) -> str | None:
+    """Generic extraction-quality gate (HARDENED:grill): empty-text / low text
+    density detector shared by every handler. Strips table-unparsed fences and
+    page markers before counting so a document that is ENTIRELY scanned pages
+    or an unparsed table dump does not slip past on marker text alone.
+    Returns a quarantine reason, or ``None`` if the content passes."""
+    import re
+
+    stripped = re.sub(r"^#{1,3}\s.*$", "", markdown, flags=re.MULTILINE)
+    stripped = stripped.replace("```", "")
+    content_chars = len(stripped.strip())
+    if content_chars < min_chars:
+        return "empty_or_low_text_density"
+    return None
+
+
+_CONTROL_CHARS = re.compile(r"[\x00-\x1f\x7f]")
+
+
+def strip_control_chars(name: str) -> str:
+    """Strip control chars (incl. embedded newlines) from an untrusted name
+    (zip member, email attachment/header, HTML title, ...) before it flows
+    into generated Markdown body text or a report entry (S06 HARDENED — the
+    S05 lesson was frontmatter; a control char in body text can still forge a
+    fake heading/table row in the rendered note or a report line)."""
+    if not name:
+        return name
+    return _CONTROL_CHARS.sub("", name)
+
+
+class Handler(ABC):
+    """One handler per file extension family."""
+
+    #: lower-cased extensions this handler claims, e.g. (".pdf",)
+    extensions: tuple[str, ...] = ()
+    #: human label for capability-probe reporting, e.g. "pypdf"
+    dependency_name: str = ""
+
+    @classmethod
+    @abstractmethod
+    def available(cls) -> bool:
+        """True iff this handler's extraction dependency import-succeeds."""
+
+    @classmethod
+    @abstractmethod
+    def extract(cls, path: Path) -> ExtractResult:
+        """Extract ``path`` to Markdown. MUST NOT raise on malformed input —
+        catch and return a quarantine ``ExtractResult`` instead (a crash here
+        would abort the whole drain, not just this one file)."""

brain/ingest/handlers/docx.py

@@ -0,0 +1,78 @@
+"""DOCX handler — python-docx. Paragraphs + tables (Markdown tables, headers
+retained per HARDENED:grill — see .tables.rows_to_markdown)."""
+from __future__ import annotations
+
+from pathlib import Path
+
+from .base import ExtractResult, Handler, density_gate
+from .tables import rows_to_markdown
+
+try:
+    import docx  # python-docx
+    _HAS_DOCX = True
+except ImportError:  # pragma: no cover
+    _HAS_DOCX = False
+
+MAX_DOCX_BYTES = 100 * 1024 * 1024
+
+
+class DocxHandler(Handler):
+    extensions = (".docx",)
+    dependency_name = "python-docx"
+
+    @classmethod
+    def available(cls) -> bool:
+        return _HAS_DOCX
+
+    @classmethod
+    def extract(cls, path: Path) -> ExtractResult:
+        if not _HAS_DOCX:
+            return ExtractResult.quarantine("missing_dependency:python-docx")
+        try:
+            size = path.stat().st_size
+        except OSError:
+            size = 0
+        if size > MAX_DOCX_BYTES:
+            return ExtractResult.quarantine("file_too_large")
+        try:
+            document = docx.Document(str(path))
+        except Exception as exc:
+            return ExtractResult.quarantine(
+                "docx_extraction_error", warnings=[f"{type(exc).__name__}: {exc}"]
+            )
+
+        parts: list[str] = []
+        try:
+            # Walk body children in document order so tables land inline with
+            # the surrounding prose rather than all at the end.
+            body = document.element.body
+            table_by_elem = {t._tbl: t for t in document.tables}
+            # C10: re-scanning document.paragraphs per body child was O(n^2).
+            # Build the element->paragraph lookup once, like table_by_elem.
+            para_by_elem = {p._p: p for p in document.paragraphs}
+            for child in body.iterchildren():
+                tag = child.tag.rsplit("}", 1)[-1]
+                if tag == "p":
+                    p = para_by_elem.get(child)
+                    if p is not None:
+                        text = p.text.strip()
+                        if text:
+                            if p.style and p.style.name and p.style.name.startswith("Heading"):
+                                parts.append(f"## {text}\n")
+                            else:
+                                parts.append(f"{text}\n")
+                elif tag == "tbl":
+                    tbl = table_by_elem.get(child)
+                    if tbl is not None:
+                        rows = [[c.text for c in row.cells] for row in tbl.rows]
+                        parts.append(rows_to_markdown(rows))
+        except Exception as exc:
+            return ExtractResult.quarantine(
+                "docx_extraction_error", warnings=[f"{type(exc).__name__}: {exc}"]
+            )
+
+        body_md = "\n".join(p for p in parts if p.strip())
+        reason = density_gate(body_md)
+        if reason:
+            return ExtractResult.quarantine(reason)
+        return ExtractResult(markdown=body_md, metadata={"tables": len(document.tables)})

brain/ingest/handlers/email.py

@@ -0,0 +1,228 @@
+"""Email (.eml, RFC 5322) handler — stdlib only (``email`` + ``html.parser``,
+mirrors the sha256-verified reference-vault reference, ADR-0003 Appendix B).
+Produces headers + body + an attachment manifest; each attachment's bytes are
+returned via ``metadata["nested"]`` so the orchestrator (pipeline.py
+``_process_nested``) re-enters the dispatcher for each one — bounded by
+``MAX_ATTACHMENTS``/``MAX_ATTACHMENT_TOTAL_BYTES`` here, and by depth +
+a shared byte/count budget at the pipeline layer (defense in depth against a
+crafted attachment-of-attachment chain)."""
+from __future__ import annotations
+
+import email
+import email.policy
+import email.utils
+import html.parser
+import re
+from pathlib import Path
+from typing import Optional
+
+from .base import ExtractResult, Handler, density_gate, strip_control_chars
+
+MAX_EML_BYTES = 50 * 1024 * 1024
+MAX_ATTACHMENTS = 50
+MAX_ATTACHMENT_TOTAL_BYTES = 200 * 1024 * 1024  # matches pipeline.MAX_INGEST_BYTES
+
+_EM_DASH = "—"
+
+
+class _HtmlStripper(html.parser.HTMLParser):
+    """Minimal stdlib HTML->text fallback for a text/html-only body (no
+    BeautifulSoup dependency needed for this narrow use)."""
+
+    _BLOCK = {"p", "div", "br", "li", "tr", "h1", "h2", "h3", "h4", "h5", "h6",
+              "blockquote", "pre", "hr"}
+    _SKIP = {"script", "style", "head", "meta", "link"}
+
+    def __init__(self) -> None:
+        super().__init__(convert_charrefs=True)
+        self._chunks: list[str] = []
+        self._skip = 0
+
+    def handle_starttag(self, tag: str, attrs: list) -> None:
+        if tag in self._SKIP:
+            self._skip += 1
+        elif tag in self._BLOCK:
+            self._chunks.append("\n")
+
+    def handle_endtag(self, tag: str) -> None:
+        if tag in self._SKIP and self._skip > 0:
+            self._skip -= 1
+        elif tag in self._BLOCK:
+            self._chunks.append("\n")
+
+    def handle_data(self, data: str) -> None:
+        if not self._skip:
+            self._chunks.append(data)
+
+    def get_text(self) -> str:
+        joined = "".join(self._chunks)
+        joined = re.sub(r"\n{3,}", "\n\n", joined)
+        return "\n".join(line.rstrip() for line in joined.split("\n")).strip()
+
+
+def _strip_html(raw: str) -> str:
+    stripper = _HtmlStripper()
+    try:
+        stripper.feed(raw)
+        stripper.close()
+    except Exception:
+        return ""
+    return stripper.get_text()
+
+
+def _decode_header(raw: object) -> str:
+    return str(raw).strip() if raw else ""
+
+
+def _addr_list(raw: str) -> list[str]:
+    if not raw:
+        return []
+    out: list[str] = []
+    for name, addr in email.utils.getaddresses([raw]):
+        name, addr = name.strip(), addr.strip()
+        if name and addr:
+            out.append(f"{name} <{addr}>")
+        elif addr:
+            out.append(addr)
+        elif name:
+            out.append(name)
+    return out
+
+
+def _sent_date_iso(raw: str) -> Optional[str]:
+    if not raw:
+        return None
+    try:
+        dt = email.utils.parsedate_to_datetime(raw)
+        return dt.isoformat() if dt is not None else None
+    except Exception:
+        return None
+
+
+def _extract_body(msg: "email.message.Message") -> tuple[str, list[str]]:
+    warnings: list[str] = []
+    text_part = html_part = None
+    for part in msg.walk():
+        if part.is_multipart() or part.get_content_disposition() == "attachment":
+            continue
+        ctype = part.get_content_type()
+        if ctype == "text/plain" and text_part is None:
+            text_part = part
+        elif ctype == "text/html" and html_part is None:
+            html_part = part
+
+    if text_part is not None:
+        try:
+            return text_part.get_content().strip(), warnings
+        except Exception:
+            payload = text_part.get_payload(decode=True) or b""
+            charset = text_part.get_content_charset() or "utf-8"
+            return payload.decode(charset, errors="replace").strip(), warnings
+
+    if html_part is not None:
+        try:
+            raw_html = html_part.get_content()
+        except Exception:
+            payload = html_part.get_payload(decode=True) or b""
+            charset = html_part.get_content_charset() or "utf-8"
+            raw_html = payload.decode(charset, errors="replace")
+        warnings.append("html_only_fallback: no text/plain part, stripped HTML")
+        return _strip_html(raw_html), warnings
+
+    warnings.append("no_body_part: neither text/plain nor text/html present")
+    return "", warnings
+
+
+class EmailHandler(Handler):
+    extensions = (".eml",)
+    dependency_name = "stdlib"
+
+    @classmethod
+    def available(cls) -> bool:
+        return True
+
+    @classmethod
+    def extract(cls, path: Path) -> ExtractResult:
+        try:
+            size = path.stat().st_size
+        except OSError:
+            size = 0
+        if size > MAX_EML_BYTES:
+            return ExtractResult.quarantine(
+                "file_too_large", warnings=[f"{size} bytes exceeds cap {MAX_EML_BYTES}"]
+            )
+        try:
+            raw = path.read_bytes()
+        except OSError as exc:
+            return ExtractResult.quarantine("eml_read_error", warnings=[f"{type(exc).__name__}: {exc}"])
+        try:
+            msg = email.message_from_bytes(raw, policy=email.policy.default)
+        except Exception as exc:
+            return ExtractResult.quarantine("eml_parse_error", warnings=[f"{type(exc).__name__}: {exc}"])
+
+        subject = strip_control_chars(_decode_header(msg.get("Subject")))
+        from_addrs = _addr_list(_decode_header(msg.get("From")))
+        to_addrs = _addr_list(_decode_header(msg.get("To")))
+        cc_addrs = _addr_list(_decode_header(msg.get("Cc")))
+        sent_raw = _decode_header(msg.get("Date"))
+        sent_iso = _sent_date_iso(sent_raw)
+        body_text, warnings = _extract_body(msg)
+
+        try:
+            attachments = list(msg.iter_attachments())
+        except Exception:
+            attachments = []
+        if len(attachments) > MAX_ATTACHMENTS:
+            warnings.append(f"attachments_truncated: {len(attachments)} found, cap {MAX_ATTACHMENTS}")
+            attachments = attachments[:MAX_ATTACHMENTS]
+
+        nested: list[dict] = []
+        attach_meta: list[tuple[str, str, int]] = []
+        total_bytes = 0
+        for idx, part in enumerate(attachments, start=1):
+            name = strip_control_chars(part.get_filename() or f"attachment_{idx}.bin")
+            try:
+                data = part.get_payload(decode=True) or b""
+            except Exception as exc:
+                warnings.append(f"attachment_decode_failed:{name}:{type(exc).__name__}")
+                continue
+            if total_bytes + len(data) > MAX_ATTACHMENT_TOTAL_BYTES:
+                warnings.append(f"attachment_byte_cap_reached: stopped before {name}")
+                break
+            total_bytes += len(data)
+            attach_meta.append((name, part.get_content_type(), len(data)))
+            nested.append({"name": name, "data": data})
+
+        from_disp = "; ".join(from_addrs) if from_addrs else _EM_DASH
+        to_disp = "; ".join(to_addrs) if to_addrs else _EM_DASH
+
+        lines = [
+            "## Email metadata", "",
+            f"- **Subject:** {subject or '(no subject)'}",
+            f"- **From:** {from_disp}",
+            f"- **To:** {to_disp}",
+        ]
+        if cc_addrs:
+            lines.append(f"- **Cc:** {'; '.join(cc_addrs)}")
+        if sent_iso:
+            lines.append(f"- **Sent:** {sent_iso} (raw: {sent_raw})")
+        elif sent_raw:
+            lines.append(f"- **Sent:** {sent_raw}")
+        if attach_meta:
+            lines.append(f"- **Attachments:** {len(attach_meta)}")
+        lines += ["", "## Body", "", body_text or "*(empty body)*", ""]
+        if attach_meta:
+            lines.append("## Attachments")
+            lines.append("")
+            for name, ctype, nbytes in attach_meta:
+                lines.append(f"- `{name}` — {ctype} ({nbytes / 1024:.1f} KB)")
+            lines.append("")
+        body_md = "\n".join(lines)
+
+        reason = density_gate(body_md)
+        if reason:
+            return ExtractResult.quarantine(reason, warnings=warnings)
+        return ExtractResult(
+            markdown=body_md, warnings=warnings,
+            metadata={"nested": nested, "attachment_count": len(attach_meta), "subject": subject},
+        )

brain/ingest/handlers/html.py

@@ -0,0 +1,142 @@
+"""HTML handler — stdlib ``html.parser`` readable-text conversion (no new
+required dependency); ``lxml`` is used as an optional faster/more-robust path
+when already installed, mirroring the sha256-verified reference-vault reference
+(ADR-0003 Appendix B) which proves the stdlib fallback alone is production-
+adequate."""
+from __future__ import annotations
+
+import html as _html_stdlib
+import html.parser
+import re
+from pathlib import Path
+
+from .base import ExtractResult, Handler, density_gate
+
+MAX_HTML_BYTES = 50 * 1024 * 1024
+
+_SKIP_TAGS = frozenset({"script", "style", "noscript", "svg", "canvas", "iframe", "object", "embed"})
+_BLOCK_TAGS = frozenset({
+    "div", "p", "br", "hr", "h1", "h2", "h3", "h4", "h5", "h6",
+    "li", "dt", "dd", "tr", "td", "th", "section", "article", "header",
+    "footer", "nav", "main", "aside", "pre", "blockquote", "table",
+    "thead", "tbody", "tfoot", "ul", "ol", "dl", "figure", "figcaption",
+})
+
+
+class _TextExtractor(html.parser.HTMLParser):
+    """Minimal HTML -> plain-text via stdlib. Void elements (br, hr, meta,
+    link, ...) fire ``handle_starttag`` but never ``handle_endtag`` — they are
+    deliberately absent from ``_SKIP_TAGS`` so ``_skip`` can never be left
+    incremented forever by one."""
+
+    def __init__(self) -> None:
+        super().__init__(convert_charrefs=True)
+        self._chunks: list[str] = []
+        self._skip = 0
+
+    def handle_starttag(self, tag: str, attrs: list) -> None:
+        tag = tag.lower()
+        if tag in _SKIP_TAGS:
+            self._skip += 1
+        if tag in _BLOCK_TAGS:
+            self._chunks.append("\n")
+
+    def handle_endtag(self, tag: str) -> None:
+        tag = tag.lower()
+        if tag in _SKIP_TAGS:
+            self._skip = max(0, self._skip - 1)
+        if tag in _BLOCK_TAGS:
+            self._chunks.append("\n")
+
+    def handle_data(self, data: str) -> None:
+        if not self._skip:
+            self._chunks.append(data)
+
+    def get_text(self) -> str:
+        raw = "".join(self._chunks)
+        lines = [re.sub(r"[ \t]+", " ", ln).strip() for ln in raw.splitlines()]
+        text = "\n".join(lines)
+        return re.sub(r"\n{3,}", "\n\n", text).strip()
+
+
+def _extract_title(raw_html: str) -> str | None:
+    m = re.search(r"<title[^>]*>(.*?)</title>", raw_html, re.IGNORECASE | re.DOTALL)
+    if not m:
+        return None
+    title = _html_stdlib.unescape(m.group(1))
+    title = re.sub(r"\s+", " ", title).strip()
+    return title or None
+
+
+def _extract_text(raw_html: str) -> tuple[str, list[str]]:
+    warnings: list[str] = []
+    try:
+        from lxml.html import fromstring as _fromstring
+
+        doc = _fromstring(raw_html)
+        for bad in doc.xpath("//script|//style|//noscript"):
+            bad.drop_tree()
+        text = doc.text_content()
+        text = re.sub(r"[ \t]+", " ", text)
+        text = re.sub(r"\n{3,}", "\n\n", text).strip()
+        if text:
+            return text, warnings
+        # fall through to stdlib if lxml produced nothing
+    except ImportError:
+        pass  # lxml not installed — stdlib fallback below
+    except Exception as exc:
+        warnings.append(f"lxml_parse_warning: {type(exc).__name__}: {exc}")
+
+    extractor = _TextExtractor()
+    try:
+        extractor.feed(raw_html)
+        return extractor.get_text(), warnings
+    except Exception as exc:
+        warnings.append(f"html_parse_warning: {type(exc).__name__}: {exc}")
+        text = re.sub(r"<[^>]+>", " ", raw_html)
+        return re.sub(r"\s+", " ", text).strip(), warnings
+
+
+class HtmlHandler(Handler):
+    extensions = (".html", ".htm")
+    dependency_name = "stdlib"
+
+    @classmethod
+    def available(cls) -> bool:
+        return True
+
+    @classmethod
+    def extract(cls, path: Path) -> ExtractResult:
+        try:
+            size = path.stat().st_size
+        except OSError:
+            size = 0
+        if size > MAX_HTML_BYTES:
+            return ExtractResult.quarantine(
+                "file_too_large", warnings=[f"{size} bytes exceeds cap {MAX_HTML_BYTES}"]
+            )
+        try:
+            raw = path.read_bytes()
+        except OSError as exc:
+            return ExtractResult.quarantine("html_read_error", warnings=[f"{type(exc).__name__}: {exc}"])
+
+        text_raw = None
+        for enc in ("utf-8", "latin-1"):
+            try:
+                text_raw = raw.decode(enc)
+                break
+            except UnicodeDecodeError:
+                continue
+        if text_raw is None:
+            return ExtractResult.quarantine("html_decode_error")
+
+        title = _extract_title(text_raw)
+        body, warnings = _extract_text(text_raw)
+        if not body:
+            return ExtractResult.quarantine("empty_or_low_text_density", warnings=warnings)
+
+        markdown = f"# {title}\n\n{body}\n" if title else f"{body}\n"
+        reason = density_gate(markdown)
+        if reason:
+            return ExtractResult.quarantine(reason, warnings=warnings)
+        return ExtractResult(markdown=markdown, warnings=warnings, metadata={"title": title})