brainiac-cli 0.16.0__py3-none-any.whl

brain/capture.py

@@ -0,0 +1,117 @@
+"""Capture-path frontmatter enforcement (UX-01).
+
+Both host-native clients (Codex/Claude Code/Gemini CLI) and sandboxed VM clients
+(Cowork) route through enforce() to guarantee frontmatter before any write.
+The host signs and indexes; the VM drops to capture-inbox/ unsigned and unindexed.
+
+    host:  enforce() → write_note() → incremental sync → snapshot
+    VM:    enforce() → draft_capture() (capture-inbox/; unsigned, unindexed)
+           host drain-on-invoke picks it up on the next brain run
+
+No signing key is ever read or resolved here.
+"""
+from __future__ import annotations
+
+import datetime
+import hashlib
+from typing import Any
+
+from . import frontmatter as fm
+
+REQUIRED_KEYS: tuple[str, ...] = ("id", "type", "classification", "created")
+_CAPTURE_CLASSIFICATION_DEFAULT = "Internal"
+_CAPTURE_TYPE_DEFAULT = "note"
+
+
+def _today() -> str:
+    return datetime.date.today().isoformat()
+
+
+def _derive_id(body: str) -> str:
+    h = hashlib.sha256(body.encode("utf-8")).hexdigest()[:12]
+    return f"capture-{h}"
+
+
+def enforce(content: str, *, override: dict[str, Any] | None = None) -> str:
+    """Return content with all required capture frontmatter guaranteed.
+
+    Rules:
+    - Existing keys are NEVER overwritten (additive only).
+    - ``override`` keys take precedence over both existing and defaults.
+    - Missing ``classification`` defaults to ``Internal`` so a captured note
+      is usable by default without requiring --max-tier elevation.
+    - Always sets ``status: draft`` and ``provenance.trust: untrusted`` so the
+      host drain treats it as untrusted input during ingest validation.
+    - Preserves all other existing frontmatter keys.
+
+    The result is STILL UNTRUSTED from the host's perspective until drain-on-invoke
+    signs it — the host promote step validates, signs, indexes, and updates status.
+    """
+    override = override or {}
+    meta, body = fm.parse_text(content)
+
+    nid = override.get("id") or meta.get("id") or _derive_id(body)
+    ntype = override.get("type") or meta.get("type") or _CAPTURE_TYPE_DEFAULT
+    ncls = (
+        override.get("classification")
+        or meta.get("classification")
+        or _CAPTURE_CLASSIFICATION_DEFAULT
+    )
+    ncreated = override.get("created") or meta.get("created") or _today()
+    nupdated = override.get("updated") or meta.get("updated") or _today()
+    ntitle = override.get("title") or meta.get("title") or str(nid)
+
+    block: dict[str, Any] = {
+        "id": nid,
+        "title": ntitle,
+        "type": ntype,
+        "classification": ncls,
+        "created": ncreated,
+        "updated": nupdated,
+        "status": "draft",
+        "provenance.trust": "untrusted",
+    }
+    # Preserve any other keys from the original frontmatter (non-clobbering).
+    for k, v in meta.items():
+        if k not in block:
+            block[k] = v
+
+    lines = ["---"]
+    for k, v in block.items():
+        sv = str(v)
+        # Quote values containing YAML-special characters.
+        if any(c in sv for c in (":", "#", "[", "]", "{", "}", ",")):
+            sv = f'"{sv}"'
+        lines.append(f"{k}: {sv}")
+    lines.append("---")
+    lines.append("")
+    lines.append(body.lstrip("\n"))
+
+    return "\n".join(lines)
+
+
+def validate(content: str) -> list[str]:
+    """Return a list of validation errors (empty list = valid).
+
+    Called by the host drain before signing to validate untrusted capture
+    content. Checks that required keys are present and classification is a
+    known tier.
+    """
+    from .classification import TIERS
+
+    meta, _body = fm.parse_text(content)
+    errors: list[str] = []
+
+    if not meta:
+        errors.append("no frontmatter")
+        return errors
+
+    for key in REQUIRED_KEYS:
+        if key not in meta:
+            errors.append(f"missing required key: {key}")
+
+    cls_val = str(meta.get("classification", ""))
+    if cls_val and cls_val not in TIERS:
+        errors.append(f"unknown classification: {cls_val!r} (valid: {list(TIERS)})")
+
+    return errors

brain/chunk.py

@@ -0,0 +1,249 @@
+"""Block/section chunking + Anthropic-style in-language contextual prefix (IDX-02).
+
+Two ideas, both lifting retrieval recall:
+
+1. **Chunk, don't embed whole notes.** A 4 KB note embedded as one vector blurs
+   every section together; a query that matches one paragraph competes against
+   the whole-note average. We split the body at Markdown section/block level
+   (headings first, then paragraph blocks, with a soft size budget) so each
+   chunk is a coherent unit.
+
+2. **Contextual prefix (Anthropic "Contextual Retrieval").** Each chunk is
+   prepended — BEFORE embedding — with a short blurb situating it inside its
+   note ("From note 'X' …; section: Y"). This restores the context a bare chunk
+   loses and sharply cuts misses. The blurb is written **in the note's own
+   language** (a Portuguese note gets a Portuguese blurb) so it does not pollute
+   the chunk's language; the cross-lingual bridge happens at *query* time in
+   Arctic-embed's shared multilingual vector space.
+
+The canonical task prefix (``query:`` / ``passage:``) is a model control token
+and is applied by the embedder (``brain.embed``), OUTSIDE this contextual prefix
+— and is never translated. The contextual prefix here is content, not a token.
+"""
+from __future__ import annotations
+
+import os
+import re
+from dataclasses import dataclass
+
+# Soft target / hard ceiling for a chunk, in characters. Blocks are merged up to
+# the target and a single oversized block is split at the ceiling.
+TARGET_CHARS = 900
+MAX_CHARS = 1400
+
+# Dual-granularity threshold (RET-03) — DISABLED BY DEFAULT (= 0) after the S10
+# A/B falsified it for this corpus + model. The hypothesis: a short curated note
+# split per-section scatters its cross-lingual signal across competing vectors, so
+# indexing it as ONE whole-note chunk should restore the edge Smart Connections
+# (whole-note, same e5-small) has on monolingual PT. EMPIRICAL RESULT (full e5-small
+# re-index, 2026-06-28): it did NOT help and slightly REGRESSED — monolingual_pt
+# 0.653 → 0.611, overall 0.573 → 0.553 vs the chunk index at the same zone weight.
+# Mashing a note's sections into one 450-token vector DILUTES the specific matching
+# section rather than concentrating it (the opposite of the literature's claim for
+# small models), and the corpus had few multi-section short notes anyway (only ~824
+# of 83k chunks merged). Kept as an env-gated capability for other corpora, but the
+# DEFAULT IS 0 (pure section chunking) because it is the better config here.
+# Set BRAIN_WHOLENOTE_MAX_CHARS=<chars> to re-enable. Evidence:
+# docs/operations/s10-agentic-retrieval-analysis.md.
+WHOLE_NOTE_MAX_CHARS = 0
+
+
+def _whole_note_max() -> int:
+    try:
+        return int(os.environ.get("BRAIN_WHOLENOTE_MAX_CHARS", "") or WHOLE_NOTE_MAX_CHARS)
+    except ValueError:
+        return WHOLE_NOTE_MAX_CHARS
+
+_HEADING = re.compile(r"^(#{1,6})\s+(.*)$")
+
+
+@dataclass
+class Chunk:
+    ordinal: int
+    heading: str       # nearest enclosing heading text ("" if none / preamble)
+    text: str          # the raw chunk body (no prefixes)
+    lang: str          # detected language of the chunk: "pt" | "es" | "en"
+
+    def embed_input(self, title: str, zone: str, doc_context: str = "") -> str:
+        """The exact string handed to the embedder for this chunk: the
+        in-language contextual prefix + a blank line + the chunk text.
+
+        ``doc_context`` (UPG-04, optional) is an LLM-generated ≤1-sentence
+        document-level summary that situates this chunk within its note. When
+        non-empty it is prepended (after the template prefix, before the chunk
+        text) so the embedding sees the note's overall meaning alongside the
+        chunk's specific content. Empty string = the S10 template-only path.
+        """
+        prefix = contextual_prefix(title, zone, self.heading, self.lang)
+        if doc_context:
+            prefix = prefix + " " + doc_context
+        return prefix + "\n\n" + self.text
+
+
+# --- language detection (lightweight, dependency-free) ---------------------
+# Stopword sets chosen to be discriminative between EN / PT / ES. Not a full
+# language ID — good enough to pick the contextual-prefix language.
+_PT = {
+    "de", "que", "não", "uma", "para", "com", "como", "mais", "está", "são",
+    "também", "já", "nós", "sobre", "será", "foi", "ção", "às", "então", "porque",
+}
+_ES = {
+    "de", "que", "no", "una", "para", "con", "como", "más", "está", "son",
+    "también", "ya", "nosotros", "sobre", "será", "fue", "ción", "pero", "porque",
+    "el", "los", "las", "una",
+}
+_EN = {
+    "the", "and", "of", "to", "in", "is", "are", "for", "with", "that", "this",
+    "as", "be", "on", "by", "an", "we", "it", "from", "will",
+}
+_WORD = re.compile(r"[A-Za-zÀ-ÿ]+")
+
+
+def detect_language(text: str) -> str:
+    """Return 'pt', 'es', or 'en' by discriminative-stopword frequency.
+
+    Defaults to 'en' on a tie or when there is too little signal. Accent marks
+    (ç, ã, õ, á, é) strongly bias toward PT/ES.
+    """
+    words = [w.lower() for w in _WORD.findall(text)]
+    if not words:
+        return "en"
+    scores = {
+        "pt": sum(1 for w in words if w in _PT),
+        "es": sum(1 for w in words if w in _ES),
+        "en": sum(1 for w in words if w in _EN),
+    }
+    # Portuguese-specific orthography (ã, õ, ç) is a strong PT signal.
+    if re.search(r"[ãõç]", text):
+        scores["pt"] += 3
+    # Spanish-specific (ñ, ¿, ¡)
+    if re.search(r"[ñ¿¡]", text):
+        scores["es"] += 3
+    best = max(scores, key=lambda k: scores[k])
+    # Require the winner to actually beat English by a margin, else default EN
+    # (avoids flipping an English chunk to PT on a couple of shared tokens).
+    if best != "en" and scores[best] <= scores["en"]:
+        return "en"
+    return best
+
+
+# --- in-language contextual prefix -----------------------------------------
+# Per-language templates. {title}=note title, {zone}=para zone, {heading}=section.
+_TEMPLATES = {
+    "en": "Context: from the note '{title}' (zone: {zone}).{section}",
+    "pt": "Contexto: da nota '{title}' (zona: {zone}).{section}",
+    "es": "Contexto: de la nota '{title}' (zona: {zone}).{section}",
+}
+_SECTION = {
+    "en": " Section: {heading}.",
+    "pt": " Secção: {heading}.",
+    "es": " Sección: {heading}.",
+}
+
+
+def contextual_prefix(title: str, zone: str, heading: str, lang: str) -> str:
+    lang = lang if lang in _TEMPLATES else "en"
+    section = _SECTION[lang].format(heading=heading) if heading else ""
+    return _TEMPLATES[lang].format(title=title or "(untitled)", zone=zone or "brain", section=section)
+
+
+# --- the chunker ------------------------------------------------------------
+def _split_blocks(body: str) -> list[tuple[str, str]]:
+    """Walk the body line by line, tracking the nearest heading, and group
+    contiguous non-heading lines into blocks. Returns [(heading, block_text)]."""
+    blocks: list[tuple[str, str]] = []
+    cur_heading = ""
+    buf: list[str] = []
+
+    def flush() -> None:
+        text = "\n".join(buf).strip()
+        if text:
+            blocks.append((cur_heading, text))
+        buf.clear()
+
+    for line in body.splitlines():
+        m = _HEADING.match(line)
+        if m:
+            flush()
+            cur_heading = m.group(2).strip()
+        elif line.strip() == "" and buf:
+            # paragraph boundary inside a section
+            flush()
+        else:
+            buf.append(line)
+    flush()
+    return blocks
+
+
+def _pack(heading: str, text: str, ordinal_start: int, lang_of) -> list[Chunk]:
+    """Split one oversized block at the char ceiling into sub-chunks."""
+    out: list[Chunk] = []
+    o = ordinal_start
+    if len(text) <= MAX_CHARS:
+        out.append(Chunk(o, heading, text, lang_of(text)))
+        return out
+    # greedy word-wrap into <= MAX_CHARS slices
+    words = text.split()
+    cur: list[str] = []
+    size = 0
+    for w in words:
+        if size + len(w) + 1 > MAX_CHARS and cur:
+            piece = " ".join(cur)
+            out.append(Chunk(o, heading, piece, lang_of(piece)))
+            o += 1
+            cur, size = [], 0
+        cur.append(w)
+        size += len(w) + 1
+    if cur:
+        piece = " ".join(cur)
+        out.append(Chunk(o, heading, piece, lang_of(piece)))
+    return out
+
+
+def chunk_text(body: str, *, lang_of=detect_language) -> list[Chunk]:
+    """Chunk a note body at section/block level with a soft size budget.
+
+    Strategy: split into heading-scoped blocks, then greedily merge adjacent
+    blocks under the same heading up to TARGET_CHARS, splitting any single block
+    over MAX_CHARS. Each chunk's language is detected on its own text.
+
+    Dual-granularity (RET-03): a short note (whole body ≤ ``_whole_note_max()``)
+    is returned as ONE whole-note chunk — its sections are NOT split — so its
+    cross-lingual signal stays concentrated in a single vector. See
+    ``WHOLE_NOTE_MAX_CHARS``.
+    """
+    whole_max = _whole_note_max()
+    stripped = body.strip()
+    if whole_max > 0 and stripped and len(stripped) <= whole_max:
+        return [Chunk(0, "", stripped, lang_of(stripped))]
+    blocks = _split_blocks(body)
+    if not blocks:
+        return []
+    chunks: list[Chunk] = []
+    ordinal = 0
+    pending_heading = blocks[0][0]
+    pending_text = ""
+
+    def emit(heading: str, text: str) -> None:
+        nonlocal ordinal
+        for ch in _pack(heading, text, ordinal, lang_of):
+            ch.ordinal = ordinal
+            chunks.append(ch)
+            ordinal += 1
+
+    for heading, text in blocks:
+        if heading != pending_heading:
+            if pending_text:
+                emit(pending_heading, pending_text)
+            pending_heading, pending_text = heading, text
+            continue
+        if not pending_text:
+            pending_text = text
+        elif len(pending_text) + len(text) + 2 <= TARGET_CHARS:
+            pending_text = pending_text + "\n\n" + text
+        else:
+            emit(pending_heading, pending_text)
+            pending_text = text
+    if pending_text:
+        emit(pending_heading, pending_text)
+    return chunks

brain/classification.py

@@ -0,0 +1,134 @@
+"""Classification tiers + the deny-by-default egress filter (CORE-02).
+
+This is the *egress-decision mechanism*, NOT containment. It only decides what
+the cooperative `brain` CLI is willing to print to stdout. Any file-capable
+harness can read the Markdown directly and bypass it entirely — that is why real
+containment of sensitive tiers is **workspace projection** (see
+``brain.projection``) plus the host/VM trust split, not this filter. The
+consensus-hardening tests (tests/test_direct_file_read.py) prove this distinction.
+
+Tiers, low -> high sensitivity:
+    Public < Internal < Confidential < Restricted < MNPI
+
+Default-deny (load-bearing): a note whose ``classification`` is missing, empty,
+or unrecognised is treated as MNPI (rank 4, most restrictive) at every surfacing
+boundary — fail-closed, never fail-open.
+"""
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+from typing import Iterable, Sequence
+
+TIERS: tuple[str, ...] = ("Public", "Internal", "Confidential", "Restricted", "MNPI")
+RANK: dict[str, int] = {t: i for i, t in enumerate(TIERS)}
+
+# The tier an unlabelled / unrecognised note is treated as (most restrictive).
+DEFAULT_DENY_TIER = "MNPI"
+DEFAULT_DENY_RANK = RANK[DEFAULT_DENY_TIER]
+
+# Default egress cap on the TRUSTED HOST: the full vault (owner decision,
+# 2026-07-10 — the old Internal default starved every real query: a curated
+# vault keeps its load-bearing notes at Confidential/Restricted, so the host
+# surface answered from stale low-tier scraps while competitors read
+# everything). $BRAIN_DEFAULT_MAX_TIER narrows it back (e.g. "Internal") for
+# deployments that want the conservative gate; an unrecognised value falls
+# back to the full-vault default. The untrusted VM leg does NOT inherit this:
+# role=vm resolves to VM_DEFAULT_MAX_TIER below (the trifecta break lives at
+# the role boundary, not on the owner's own host).
+DEFAULT_MAX_TIER = os.environ.get("BRAIN_DEFAULT_MAX_TIER", "MNPI")
+if DEFAULT_MAX_TIER not in RANK:
+    DEFAULT_MAX_TIER = "MNPI"
+
+# Conservative default for the untrusted read+draft leg (role=vm): surface
+# Public + Internal only unless a human explicitly elevates with --max-tier.
+VM_DEFAULT_MAX_TIER = "Internal"
+
+
+def normalize(value: object) -> str:
+    """Map a raw frontmatter value to a recognised tier, default-deny on miss."""
+    if isinstance(value, str) and value.strip() in RANK:
+        return value.strip()
+    return DEFAULT_DENY_TIER
+
+
+def rank(value: object) -> int:
+    """Effective sensitivity rank, default-deny (unlabelled -> MNPI rank)."""
+    return RANK[normalize(value)]
+
+
+def is_default_denied(value: object) -> bool:
+    """True iff the raw value would be coerced to the default-deny tier."""
+    return not (isinstance(value, str) and value.strip() in RANK)
+
+
+# Lowercased tier -> canonical, for detecting casing mistakes (F-04).
+_CANON_BY_LOWER = {t.lower(): t for t in TIERS}
+
+
+def casing_mismatch(value: object) -> str | None:
+    """If ``value`` is a KNOWN tier in the wrong case (e.g. 'internal'), return
+    its canonical form; else None.
+
+    DESIGN DECISION (F-04): the filter keeps STRICT matching — a non-canonical
+    value is default-denied (fail-closed), never silently up-ranked (which would
+    be fail-OPEN). But a wrong-case known tier is almost always an authoring slip
+    that would make the note invisible forever, so we surface it as a diagnostic
+    here (and in redaction_report) instead of letting it vanish silently. The
+    fix-at-source is tools/validate.py, which flags non-canonical casing.
+    """
+    if isinstance(value, str):
+        v = value.strip()
+        if v not in RANK and v.lower() in _CANON_BY_LOWER:
+            return _CANON_BY_LOWER[v.lower()]
+    return None
+
+
+@dataclass(frozen=True)
+class ClassificationFilter:
+    """Deny-by-default egress filter applied as the FINAL stage before stdout.
+
+    A note is surfaceable iff its effective rank <= the caller's max-tier rank.
+    Unlabelled/unrecognised -> MNPI -> only surfaceable when max_tier is MNPI
+    (the explicit human-gated path).
+    """
+
+    max_tier: str = DEFAULT_MAX_TIER
+
+    def __post_init__(self) -> None:
+        if self.max_tier not in RANK:
+            raise ValueError(
+                f"unknown max_tier {self.max_tier!r}; expected one of {TIERS}"
+            )
+
+    @property
+    def max_rank(self) -> int:
+        return RANK[self.max_tier]
+
+    def allows(self, classification: object) -> bool:
+        return rank(classification) <= self.max_rank
+
+    def filter(self, items: Iterable[dict], key: str = "classification") -> list[dict]:
+        """Drop any item whose classification exceeds the cap. Pure; no mutation."""
+        return [it for it in items if self.allows(it.get(key))]
+
+    def redaction_report(self, items: Sequence[dict], key: str = "classification") -> dict:
+        """How many items were withheld and why (for an honest CLI footer)."""
+        denied = [it for it in items if not self.allows(it.get(key))]
+        default_denied = sum(1 for it in denied if is_default_denied(it.get(key)))
+        # Surface wrong-case known tiers (F-04) so they don't vanish silently.
+        casing = sorted({
+            f"{it.get(key)!r}->{c}"
+            for it in items
+            if (c := casing_mismatch(it.get(key))) is not None
+        })
+        report = {
+            "total": len(items),
+            "surfaced": len(items) - len(denied),
+            "withheld": len(denied),
+            "withheld_unlabelled_default_deny": default_denied,
+            "max_tier": self.max_tier,
+        }
+        if casing:
+            report["casing_mismatch_warnings"] = casing
+        return report