brainiac-cli 0.16.0__py3-none-any.whl

brain/ingest/handlers/image.py

@@ -0,0 +1,91 @@
+"""Image handler — Pillow for metadata, optional LOCAL-only OCR (pytesseract
++ the system tesseract binary). ADR-0003 Ruling 1(g): image OCR degrades
+gracefully when the local engine is absent; there is NO cloud OCR code path
+at all in this kernel (never optional, never a fallback) — a screenshot with
+no local OCR available simply becomes a metadata-only note, never a quarantine.
+
+ponytail: no HEIC/HEIF (would need the extra pillow-heif optional dep) — add
+if a real HEIC drop shows up; Pillow alone already covers the common formats.
+"""
+from __future__ import annotations
+
+from pathlib import Path
+
+from .base import ExtractResult, Handler, density_gate
+
+try:
+    from PIL import Image
+    _HAS_PIL = True
+except ImportError:  # pragma: no cover - exercised via degraded-deps test
+    _HAS_PIL = False
+
+try:
+    import pytesseract
+    _HAS_PYTESSERACT = True
+except ImportError:  # pragma: no cover
+    _HAS_PYTESSERACT = False
+
+MAX_IMAGE_BYTES = 100 * 1024 * 1024
+
+
+def _ocr(img: "Image.Image") -> tuple[str, list[str]]:
+    """LOCAL-only OCR. Never raises: missing binding, missing tesseract
+    binary (pytesseract.TesseractNotFoundError), or any other engine failure
+    all degrade to a metadata-only body — there is no cloud fallback to
+    reach for, so failure here is never fatal to the ingest."""
+    if not _HAS_PYTESSERACT:
+        return "", ["ocr_unavailable: pytesseract not installed (metadata-only)"]
+    try:
+        text = pytesseract.image_to_string(img)
+        return text.strip(), []
+    except Exception as exc:
+        return "", [f"ocr_unavailable: {type(exc).__name__}: {exc}"]
+
+
+class ImageHandler(Handler):
+    extensions = (".png", ".jpg", ".jpeg", ".webp", ".gif", ".bmp", ".tiff")
+    dependency_name = "Pillow"
+
+    @classmethod
+    def available(cls) -> bool:
+        return _HAS_PIL
+
+    @classmethod
+    def extract(cls, path: Path) -> ExtractResult:
+        if not _HAS_PIL:
+            return ExtractResult.quarantine("missing_dependency:Pillow")
+        try:
+            size = path.stat().st_size
+        except OSError:
+            size = 0
+        if size > MAX_IMAGE_BYTES:
+            return ExtractResult.quarantine(
+                "file_too_large", warnings=[f"{size} bytes exceeds cap {MAX_IMAGE_BYTES}"]
+            )
+        try:
+            with Image.open(path) as img:
+                img.load()  # force decode now, inside the try
+                fmt = img.format or path.suffix.lstrip(".").upper() or "image"
+                width, height = img.size
+                mode = img.mode
+                ocr_text, warnings = _ocr(img)
+        except Exception as exc:
+            return ExtractResult.quarantine(
+                "image_open_error", warnings=[f"{type(exc).__name__}: {exc}"]
+            )
+
+        body = (
+            "## OCR (verbatim)\n\n"
+            f"{ocr_text if ocr_text else '[no text detected]'}\n\n"
+            "## Image metadata\n\n"
+            f"- **Format:** {fmt}\n"
+            f"- **Dimensions:** {width} x {height} px\n"
+            f"- **Mode:** {mode}\n"
+        )
+        reason = density_gate(body)
+        if reason:
+            return ExtractResult.quarantine(reason, warnings=warnings)
+        return ExtractResult(
+            markdown=body, warnings=warnings,
+            metadata={"format": fmt, "width": width, "height": height, "ocr": bool(ocr_text)},
+        )

brain/ingest/handlers/pdf.py

@@ -0,0 +1,99 @@
+"""PDF handler — pypdf only (ADR-0003 Ruling 1(g): pure-ish Python, no system
+binaries; no poppler/pdfplumber acceleration — out of scope for this session,
+add if a size/latency ceiling is ever measured to need it).
+
+ponytail: no image extraction, no poppler fast-lane for huge PDFs (the
+reference vault's biggest complexity driver) — this session's deliverable is
+faithful TEXT extraction with a quality gate, not image/asset pipelines. Add
+poppler-accelerated large-PDF lane if a real corpus ever needs it.
+"""
+from __future__ import annotations
+
+from pathlib import Path
+
+from .base import ExtractResult, Handler, density_gate
+
+try:
+    from pypdf import PdfReader
+    from pypdf.errors import PdfReadError
+    _HAS_PYPDF = True
+except ImportError:  # pragma: no cover - exercised via degraded-deps test
+    _HAS_PYPDF = False
+
+_MIN_PAGE_CHARS = 5
+# HARDENED:codex — max-size cap so a pathological file can't hang extraction
+# or blow memory before it ever reaches the signed write path.
+MAX_PDF_BYTES = 200 * 1024 * 1024  # 200 MB
+
+
+class PdfHandler(Handler):
+    extensions = (".pdf",)
+    dependency_name = "pypdf"
+
+    @classmethod
+    def available(cls) -> bool:
+        return _HAS_PYPDF
+
+    @classmethod
+    def extract(cls, path: Path) -> ExtractResult:
+        if not _HAS_PYPDF:
+            return ExtractResult.quarantine("missing_dependency:pypdf")
+        try:
+            size = path.stat().st_size
+        except OSError:
+            size = 0
+        if size > MAX_PDF_BYTES:
+            return ExtractResult.quarantine(
+                "file_too_large",
+                warnings=[f"pdf {size} bytes exceeds cap {MAX_PDF_BYTES}"],
+            )
+        try:
+            reader = PdfReader(str(path))
+        except Exception as exc:
+            return ExtractResult.quarantine(
+                "pdf_extraction_error", warnings=[f"{type(exc).__name__}: {exc}"]
+            )
+        if reader.is_encrypted:
+            # Pre-sign guard (HARDENED:grill) — never sign a garbage/opaque
+            # extraction of a password-protected file.
+            return ExtractResult.quarantine("pdf_encrypted")
+
+        scanned_pages: list[int] = []
+        sections: list[str] = []
+        try:
+            total = len(reader.pages)
+            for i, page in enumerate(reader.pages, start=1):
+                text = (page.extract_text() or "").strip()
+                if len(text) < _MIN_PAGE_CHARS:
+                    scanned_pages.append(i)
+                    sections.append(f"## Page {i} (scanned — no text extracted)\n")
+                else:
+                    sections.append(f"## Page {i}\n\n{text}\n")
+        except PdfReadError as exc:
+            return ExtractResult.quarantine(
+                "pdf_extraction_error", warnings=[f"{type(exc).__name__}: {exc}"]
+            )
+        except Exception as exc:
+            return ExtractResult.quarantine(
+                "pdf_extraction_error", warnings=[f"{type(exc).__name__}: {exc}"]
+            )
+
+        if total == 0 or len(scanned_pages) == total:
+            return ExtractResult.quarantine(
+                "pdf_no_text_layer",
+                warnings=[f"{len(scanned_pages)}/{total} pages had no extractable text"],
+            )
+
+        body = "\n".join(sections)
+        reason = density_gate(body)
+        if reason:
+            return ExtractResult.quarantine(reason)
+
+        warnings = []
+        if scanned_pages:
+            warnings.append(f"scanned_pages: {scanned_pages}")
+        return ExtractResult(
+            markdown=body,
+            warnings=warnings,
+            metadata={"page_count": total, "scanned_pages": scanned_pages},
+        )

brain/ingest/handlers/pptx.py

@@ -0,0 +1,69 @@
+"""PPTX handler — python-pptx. One `## Slide N` section per slide; text frames
++ tables (Markdown tables, headers retained)."""
+from __future__ import annotations
+
+from pathlib import Path
+
+from .base import ExtractResult, Handler, density_gate
+from .tables import rows_to_markdown
+
+try:
+    from pptx import Presentation
+    _HAS_PPTX = True
+except ImportError:  # pragma: no cover
+    _HAS_PPTX = False
+
+MAX_PPTX_BYTES = 150 * 1024 * 1024
+
+
+class PptxHandler(Handler):
+    extensions = (".pptx",)
+    dependency_name = "python-pptx"
+
+    @classmethod
+    def available(cls) -> bool:
+        return _HAS_PPTX
+
+    @classmethod
+    def extract(cls, path: Path) -> ExtractResult:
+        if not _HAS_PPTX:
+            return ExtractResult.quarantine("missing_dependency:python-pptx")
+        try:
+            size = path.stat().st_size
+        except OSError:
+            size = 0
+        if size > MAX_PPTX_BYTES:
+            return ExtractResult.quarantine("file_too_large")
+        try:
+            prs = Presentation(str(path))
+        except Exception as exc:
+            return ExtractResult.quarantine(
+                "pptx_extraction_error", warnings=[f"{type(exc).__name__}: {exc}"]
+            )
+
+        sections: list[str] = []
+        slide_count = 0
+        try:
+            for i, slide in enumerate(prs.slides, start=1):
+                slide_count = i
+                lines = [f"## Slide {i}\n"]
+                for shape in slide.shapes:
+                    if shape.has_table:
+                        tbl = shape.table
+                        rows = [[c.text for c in row.cells] for row in tbl.rows]
+                        lines.append(rows_to_markdown(rows))
+                    elif shape.has_text_frame:
+                        text = shape.text_frame.text.strip()
+                        if text:
+                            lines.append(text + "\n")
+                sections.append("\n".join(lines))
+        except Exception as exc:
+            return ExtractResult.quarantine(
+                "pptx_extraction_error", warnings=[f"{type(exc).__name__}: {exc}"]
+            )
+
+        body = "\n".join(sections)
+        reason = density_gate(body)
+        if reason:
+            return ExtractResult.quarantine(reason)
+        return ExtractResult(markdown=body, metadata={"slide_count": slide_count})

brain/ingest/handlers/tables.py

@@ -0,0 +1,41 @@
+"""Shared table-to-Markdown reconstruction (HARDENED:grill).
+
+Header loss silently corrupts facts at low character-error rates (a table
+flattened to prose loses the row/column association entirely). Rule: preserve
+structure as a Markdown table with headers retained; a table that cannot be
+reconstructed with headers is emitted as a fenced block flagged
+``table-unparsed`` instead of being flattened to prose.
+"""
+from __future__ import annotations
+
+
+def _cell(v: object) -> str:
+    s = "" if v is None else str(v)
+    return s.replace("|", "\\|").replace("\n", " ").strip()
+
+
+def rows_to_markdown(rows: list[list[object]], *, label: str = "table") -> str:
+    """Render ``rows`` (first row = header) as a Markdown table, or a fenced
+    ``table-unparsed`` block if headers can't be trusted (empty header row,
+    ragged column counts, or zero data rows)."""
+    rows = [r for r in rows if any(_cell(c) for c in r)]  # drop fully-blank rows
+    if not rows:
+        return ""
+    header = [_cell(c) for c in rows[0]]
+    ncols = len(header)
+    unparsed = (
+        ncols == 0
+        or not any(header)
+        or len(rows) < 2
+        or any(len(r) != ncols for r in rows[1:])
+    )
+    if unparsed:
+        raw = "\n".join(" | ".join(_cell(c) for c in r) for r in rows)
+        return f"```table-unparsed\n{raw}\n```\n"
+    lines = [
+        "| " + " | ".join(header) + " |",
+        "| " + " | ".join(["---"] * ncols) + " |",
+    ]
+    for r in rows[1:]:
+        lines.append("| " + " | ".join(_cell(c) for c in r) + " |")
+    return "\n".join(lines) + "\n"

brain/ingest/handlers/text.py

@@ -0,0 +1,43 @@
+"""Plain-text / Markdown pass-through handler."""
+from __future__ import annotations
+
+from pathlib import Path
+
+from .base import ExtractResult, Handler, density_gate
+
+_ENCODINGS = ("utf-8", "utf-8-sig", "latin-1")
+
+
+class TextHandler(Handler):
+    extensions = (".txt", ".md", ".markdown", ".csv")
+    dependency_name = "stdlib"
+
+    @classmethod
+    def available(cls) -> bool:
+        return True
+
+    @classmethod
+    def extract(cls, path: Path) -> ExtractResult:
+        # C2(b): read_bytes() used to be unguarded — an OSError (permission
+        # denied, disk error, vanished file) propagated as a raw exception out
+        # of a handler, which the contract (see handlers/base.py Handler.extract
+        # docstring) forbids: a crash here must never abort the whole drain.
+        try:
+            raw = path.read_bytes()
+        except OSError as exc:
+            return ExtractResult.quarantine(
+                "text_read_error", warnings=[f"{type(exc).__name__}: {exc}"]
+            )
+        text: str | None = None
+        for enc in _ENCODINGS:
+            try:
+                text = raw.decode(enc)
+                break
+            except UnicodeDecodeError:
+                continue
+        if text is None:
+            return ExtractResult.quarantine("text_decode_error")
+        reason = density_gate(text)
+        if reason:
+            return ExtractResult.quarantine(reason)
+        return ExtractResult(markdown=text, metadata={"encoding": "utf-8"})

brain/ingest/handlers/xlsx.py

@@ -0,0 +1,100 @@
+"""XLSX handler — openpyxl. One `## Sheet: <name>` section per sheet, rendered
+as a Markdown table (headers retained). Formula collapse (HARDENED, per the
+session brief): prefer the last-computed CACHED value (``data_only=True``); a
+formula with no cached value (never opened in Excel) falls back to the raw
+formula text tagged `(formula, uncomputed)` rather than crashing or silently
+dropping the cell."""
+from __future__ import annotations
+
+from pathlib import Path
+
+from .base import ExtractResult, Handler, density_gate
+from .tables import rows_to_markdown
+
+try:
+    import openpyxl
+    _HAS_OPENPYXL = True
+except ImportError:  # pragma: no cover
+    _HAS_OPENPYXL = False
+
+MAX_XLSX_BYTES = 100 * 1024 * 1024
+MAX_ROWS_PER_SHEET = 20_000  # cap runaway sheets rather than hang
+
+
+class XlsxHandler(Handler):
+    extensions = (".xlsx",)
+    dependency_name = "openpyxl"
+
+    @classmethod
+    def available(cls) -> bool:
+        return _HAS_OPENPYXL
+
+    @classmethod
+    def extract(cls, path: Path) -> ExtractResult:
+        if not _HAS_OPENPYXL:
+            return ExtractResult.quarantine("missing_dependency:openpyxl")
+        try:
+            size = path.stat().st_size
+        except OSError:
+            size = 0
+        if size > MAX_XLSX_BYTES:
+            return ExtractResult.quarantine("file_too_large")
+
+        try:
+            wb_values = openpyxl.load_workbook(str(path), data_only=True, read_only=True)
+        except Exception as exc:
+            return ExtractResult.quarantine(
+                "xlsx_extraction_error", warnings=[f"{type(exc).__name__}: {exc}"]
+            )
+        try:
+            wb_formulas = openpyxl.load_workbook(str(path), data_only=False, read_only=True)
+        except Exception:
+            wb_formulas = None
+
+        sections: list[str] = []
+        warnings: list[str] = []
+        try:
+            for name in wb_values.sheetnames:
+                ws_v = wb_values[name]
+                ws_f = wb_formulas[name] if wb_formulas is not None else None
+                rows: list[list[object]] = []
+                # C9: ws_f.cell(row=, column=) is RANDOM ACCESS on a
+                # read_only workbook — each call silently re-parses the
+                # whole sheet from the start, and it fired for every single
+                # empty cell (quadratic). Iterate both workbooks' rows in
+                # lockstep instead (each row read exactly once), only
+                # consulting the formula row when the value cell is empty.
+                if ws_f is not None:
+                    row_pairs = zip(ws_v.iter_rows(), ws_f.iter_rows())
+                else:
+                    row_pairs = ((row, None) for row in ws_v.iter_rows())
+                for r_idx, (row, f_row) in enumerate(row_pairs, start=1):
+                    if r_idx > MAX_ROWS_PER_SHEET:
+                        warnings.append(f"sheet {name!r} truncated at {MAX_ROWS_PER_SHEET} rows")
+                        break
+                    out_row = []
+                    for c_idx, cell in enumerate(row):
+                        val = cell.value
+                        if val is None and f_row is not None and c_idx < len(f_row):
+                            f_val = f_row[c_idx].value
+                            if isinstance(f_val, str) and f_val.startswith("="):
+                                val = f"{f_val} (formula, uncomputed)"
+                        out_row.append(val)
+                    rows.append(out_row)
+                sections.append(f"## Sheet: {name}\n\n" + rows_to_markdown(rows))
+        except Exception as exc:
+            return ExtractResult.quarantine(
+                "xlsx_extraction_error", warnings=[f"{type(exc).__name__}: {exc}"]
+            )
+        finally:
+            wb_values.close()
+            if wb_formulas is not None:
+                wb_formulas.close()
+
+        body = "\n".join(sections)
+        reason = density_gate(body)
+        if reason:
+            return ExtractResult.quarantine(reason)
+        return ExtractResult(
+            markdown=body, warnings=warnings, metadata={"sheet_count": len(sections)}
+        )

brain/ingest/handlers/zip.py

@@ -0,0 +1,163 @@
+"""ZIP handler — bounded, Zip-Slip-hardened member expansion. No file in a
+ZIP is EVER extracted to a path derived from its own name: every member's
+bytes are read in-memory here and handed to the pipeline as
+``metadata["nested"]`` entries (name, data); the pipeline
+(``pipeline._process_nested``) is the only thing that ever writes them to
+disk, and it does so under a SYNTHETIC generated name, never the archive's
+own path (S06 HARDENED:codex-verify-r2).
+
+Caps are checked from the central directory (``ZipInfo.file_size`` /
+``.external_attr``) BEFORE any member is decompressed, and each member's
+actual decompressed byte count is also counted DURING extraction (a
+malformed/lying declared size must not buy a bigger decompression than the
+cap allows) — "before/during, never after" per the S06 brief.
+"""
+from __future__ import annotations
+
+import posixpath
+import re
+import zipfile
+from pathlib import Path
+
+from .base import ExtractResult, Handler, density_gate, strip_control_chars
+
+MAX_MEMBERS = 500
+MAX_MEMBER_BYTES = 200 * 1024 * 1024            # per-member cap (matches pipeline.MAX_INGEST_BYTES)
+MAX_TOTAL_DECLARED_BYTES = 500 * 1024 * 1024    # sum of DECLARED (pre-decompress) sizes
+
+_WINDOWS_DRIVE = re.compile(r"^[A-Za-z]:")
+_S_IFLNK = 0xA000
+_S_IFREG = 0x8000
+
+
+def _unsafe_zip_member_reason(info: "zipfile.ZipInfo") -> str | None:
+    """Zip-Slip hardening: reject absolute paths, ``..`` traversal, a
+    Windows-drive-rooted name, and symlink/hardlink/non-regular members —
+    BEFORE any member is opened for reading. A zip's own path separator is
+    always ``/`` per the spec; a member trying to look like a Windows
+    absolute path (drive letter, or a literal backslash root) is just as
+    hostile as a POSIX one."""
+    name = info.filename
+    if not name or not name.strip():
+        return "empty_member_name"
+    normalized = name.replace("\\", "/")
+    if normalized.startswith("/") or posixpath.isabs(normalized):
+        return "absolute_path"
+    if _WINDOWS_DRIVE.match(normalized):
+        return "windows_drive_path"
+    if any(part == ".." for part in normalized.split("/")):
+        return "path_traversal"
+    # external_attr's high 16 bits carry unix st_mode ONLY when the member was
+    # created on a unix host (create_system == 3); on other systems (e.g. 0 =
+    # FAT/Windows) those bits are meaningless and must not be interpreted.
+    if info.create_system == 3:
+        mode = (info.external_attr >> 16) & 0xF000
+        if mode == _S_IFLNK:
+            return "symlink_member"
+        if mode not in (0, _S_IFREG):
+            return "non_regular_member"
+    return None
+
+
+def _read_member_bounded(zf: "zipfile.ZipFile", info: "zipfile.ZipInfo", cap: int) -> bytes | None:
+    """Stream-decompress ``info`` in chunks, counting REAL output bytes as
+    they arrive. Returns ``None`` if the actual decompressed size exceeds
+    ``cap`` (defends against a declared ``file_size`` that lies) instead of
+    ever fully materializing an over-cap member."""
+    chunks: list[bytes] = []
+    total = 0
+    with zf.open(info) as fh:
+        while True:
+            chunk = fh.read(1024 * 1024)
+            if not chunk:
+                break
+            total += len(chunk)
+            if total > cap:
+                return None
+            chunks.append(chunk)
+    return b"".join(chunks)
+
+
+class ZipHandler(Handler):
+    extensions = (".zip",)
+    dependency_name = "stdlib"
+
+    @classmethod
+    def available(cls) -> bool:
+        return True
+
+    @classmethod
+    def extract(cls, path: Path) -> ExtractResult:
+        try:
+            zf = zipfile.ZipFile(path)
+        except zipfile.BadZipFile as exc:
+            return ExtractResult.quarantine("zip_corrupt", warnings=[f"{type(exc).__name__}: {exc}"])
+        except OSError as exc:
+            return ExtractResult.quarantine("zip_read_error", warnings=[f"{type(exc).__name__}: {exc}"])
+
+        with zf:
+            infos = [i for i in zf.infolist() if not i.is_dir()]
+            if not infos:
+                return ExtractResult.quarantine("zip_empty")
+            if len(infos) > MAX_MEMBERS:
+                return ExtractResult.quarantine(
+                    "zip_too_many_members",
+                    warnings=[f"{len(infos)} members exceeds cap {MAX_MEMBERS}"],
+                )
+
+            # Pre-scan BEFORE any decompression: Zip-Slip safety + declared-size caps.
+            total_declared = 0
+            for info in infos:
+                reason = _unsafe_zip_member_reason(info)
+                if reason:
+                    return ExtractResult.quarantine(
+                        "zip_unsafe_member",
+                        warnings=[f"{strip_control_chars(info.filename)}: {reason}"],
+                    )
+                if info.file_size > MAX_MEMBER_BYTES:
+                    return ExtractResult.quarantine(
+                        "zip_member_too_large",
+                        warnings=[f"{strip_control_chars(info.filename)}: declared {info.file_size} bytes"],
+                    )
+                total_declared += info.file_size
+                if total_declared > MAX_TOTAL_DECLARED_BYTES:
+                    return ExtractResult.quarantine(
+                        "zip_bomb_suspected",
+                        warnings=[f"declared total {total_declared} bytes exceeds cap {MAX_TOTAL_DECLARED_BYTES}"],
+                    )
+
+            nested: list[dict] = []
+            listing: list[str] = []
+            for info in infos:
+                try:
+                    data = _read_member_bounded(zf, info, MAX_MEMBER_BYTES)
+                except Exception as exc:
+                    return ExtractResult.quarantine(
+                        "zip_extraction_error",
+                        warnings=[f"{strip_control_chars(info.filename)}: {type(exc).__name__}: {exc}"],
+                    )
+                if data is None:
+                    return ExtractResult.quarantine(
+                        "zip_bomb_suspected",
+                        warnings=[f"{strip_control_chars(info.filename)}: decompressed beyond declared size"],
+                    )
+                # Member names NEVER become filesystem paths directly — only
+                # the sanitized BASENAME (no directory components) is even
+                # offered to the pipeline, which further wraps it in a
+                # synthetic generated filename before it ever touches disk.
+                basename = strip_control_chars(Path(info.filename).name) or "member"
+                nested.append({"name": basename, "data": data})
+                listing.append(f"- `{basename}` ({len(data)} bytes)")
+
+        body = "## Archive contents\n\n" + "\n".join(listing) + "\n"
+        # ponytail: min_chars=1, not the shared 40-char default — this body is
+        # a member LISTING (quantity of entries, not extracted prose), so the
+        # prose-density threshold doesn't fit; a non-empty listing already
+        # proves real content. Still routed through the same shared gate
+        # function per the "same gate applies to every handler" rule, just
+        # tuned to this content's shape — this only guards a truly-empty
+        # listing that somehow slipped past the `zip_empty` check above.
+        reason = density_gate(body, min_chars=1)
+        if reason:
+            return ExtractResult.quarantine(reason)
+        return ExtractResult(markdown=body, metadata={"nested": nested, "member_count": len(nested)})