secondbrain-py 0.2.1__py3-none-any.whl

brain/backfill/search_extras.py

@@ -0,0 +1,141 @@
+"""Backfill ``chunks.title_text`` / ``tags_text`` / ``search_extras`` after migration 009."""
+from dataclasses import dataclass
+
+import psycopg
+
+from ..ingest.sub_tokens import extract_sub_tokens
+
+
+@dataclass
+class BackfillReport:
+    """Outcome of :func:`run`.
+
+    ``stage_a_rows`` is the number of chunk rows updated by the SQL
+    denormalization pass (title/tags from the parent document). ``stage_b_rows``
+    is the number of chunk rows whose ``search_extras`` was written by the
+    Python recompute pass — only rows whose computed value differs from the
+    stored value get rewritten, so a converged corpus reports 0 here even
+    when called repeatedly. ``total_chunks`` is the row count of the
+    ``chunks`` table at the start of the run; provided for observability so
+    operators can sanity-check the proportions.
+    """
+
+    stage_a_rows: int
+    stage_b_rows: int
+    total_chunks: int
+
+
+# Stage A — denormalize ``documents.title`` and ``documents.tags`` onto chunks.
+# ``IS DISTINCT FROM`` on both columns makes this an idempotent UPDATE; a
+# converged corpus matches zero rows on the second call. The join is on
+# ``chunks.document_id = d.id`` (FK), so every chunk has exactly one source
+# document — the UPDATE rowcount equals the count of mismatching rows.
+_STAGE_A_SQL = (
+    "UPDATE chunks "
+    "SET title_text = d.title, "
+    "    tags_text  = array_to_string(d.tags, ' ') "
+    "FROM documents d "
+    "WHERE chunks.document_id = d.id "
+    "  AND (chunks.title_text IS DISTINCT FROM d.title "
+    "       OR chunks.tags_text IS DISTINCT FROM array_to_string(d.tags, ' '))"
+)
+
+
+# Stage B reads chunks in pages of this size to keep memory bounded on a
+# 100k-chunk corpus. ``content`` is the only large column we pull, but
+# materializing all of it in one shot would cost on the order of (chunk_size
+# * row_count) of RSS — at 1k tokens (~4kB) per chunk and 100k chunks that's
+# ~400MB. Paging keeps it bounded at ~4MB per batch.
+_STAGE_B_PAGE_SIZE = 1000
+
+
+def run(conn: psycopg.Connection) -> BackfillReport:
+    """Backfill ``chunks.title_text`` / ``tags_text`` / ``search_extras``.
+
+    Two stages, both idempotent and rerunnable:
+
+    Stage A — one SQL ``UPDATE`` denormalizes ``documents.title`` and
+    ``documents.tags`` onto every chunk via the ``document_id`` FK. The
+    ``IS DISTINCT FROM`` guards mean a converged corpus is a no-op (returns
+    ``stage_a_rows=0``).
+
+    Stage B — Python loop. Reads chunks in pages of ``_STAGE_B_PAGE_SIZE``,
+    recomputes ``extract_sub_tokens(content)`` for each, and writes back only
+    when the computed value differs from the stored ``search_extras``. This
+    "compute and compare" shape (per revision #5 of the plan) restores the
+    canonical value when an operator manually edits ``search_extras`` to a
+    stale string.
+
+    Both stages run within ``conn.transaction()`` blocks. Caller is
+    responsible for connection lifecycle.
+
+    Returns a :class:`BackfillReport` with per-stage rowcounts and the
+    total chunk count observed at the start of the run.
+    """
+    total_row = conn.execute("SELECT count(*) FROM chunks").fetchone()
+    assert total_row is not None  # COUNT(*) always returns one row
+    total_chunks = int(total_row[0])
+
+    # Stage A — single SQL UPDATE in its own transaction. Empty-corpus case
+    # returns rowcount 0 naturally; no special-casing needed.
+    with conn.transaction():
+        cur = conn.execute(_STAGE_A_SQL)
+        stage_a_rows = cur.rowcount or 0
+
+    if total_chunks == 0:
+        return BackfillReport(
+            stage_a_rows=stage_a_rows,
+            stage_b_rows=0,
+            total_chunks=0,
+        )
+
+    # Stage B — page through chunks, recompute, write only on diff. We use
+    # keyset pagination on the primary key (UUID) rather than OFFSET so the
+    # page boundary stays stable when concurrent writes shift row positions.
+    # Mirrors :func:`brain.queries.iter_chunks_missing_embedding`'s pattern
+    # of casting ``id::text`` for transit and ``%s::uuid`` for the bound on
+    # the next page.
+    stage_b_rows = 0
+    last_id: str | None = None
+    while True:
+        if last_id is None:
+            page = conn.execute(
+                "SELECT id::text, content, search_extras FROM chunks "
+                "ORDER BY id LIMIT %s",
+                (_STAGE_B_PAGE_SIZE,),
+            ).fetchall()
+        else:
+            page = conn.execute(
+                "SELECT id::text, content, search_extras FROM chunks "
+                "WHERE id > %s::uuid ORDER BY id LIMIT %s",
+                (last_id, _STAGE_B_PAGE_SIZE),
+            ).fetchall()
+        if not page:
+            break
+
+        with conn.transaction():
+            for chunk_id, content, stored_extras in page:
+                computed = extract_sub_tokens(content or "")
+                # Treat NULL stored_extras and empty string as equivalent —
+                # the generated tsv column ``coalesce(...)``s NULL to '', so
+                # writing "" back when the stored value is NULL would be an
+                # observable no-op only at the row level. Skip it to keep
+                # the rowcount honest.
+                current = stored_extras or ""
+                if computed == current:
+                    continue
+                conn.execute(
+                    "UPDATE chunks SET search_extras = %s WHERE id = %s::uuid",
+                    (computed, chunk_id),
+                )
+                stage_b_rows += 1
+
+        last_id = str(page[-1][0])
+        if len(page) < _STAGE_B_PAGE_SIZE:
+            break
+
+    return BackfillReport(
+        stage_a_rows=stage_a_rows,
+        stage_b_rows=stage_b_rows,
+        total_chunks=total_chunks,
+    )

brain/backfill/source_rows.py

@@ -0,0 +1,101 @@
+"""Backfill ``documents.source_id`` for legacy file-ingested rows."""
+from dataclasses import dataclass
+
+import psycopg
+
+from ..ingest import _upsert_source
+
+
+@dataclass
+class BackfillReport:
+    """Outcome of :func:`backfill_source_rows`.
+
+    ``candidates`` is the number of rows matched by the WHERE clause, BEFORE
+    any writes. ``sources_created`` counts new ``sources`` rows inserted by
+    this run (existing rows reused via the ``(kind, external_id)`` unique
+    pair don't count). ``documents_updated`` is the number of ``documents``
+    rows whose ``source_id`` was set by this run. ``dry_run`` mirrors the
+    ``commit`` argument: ``True`` means no writes were applied (counts come
+    from the same query that would have been used for the live run).
+    """
+
+    candidates: int
+    sources_created: int
+    documents_updated: int
+    dry_run: bool
+
+
+_BACKFILL_SQL = (
+    "SELECT id, source_path FROM documents "
+    "WHERE source_id IS NULL "
+    "  AND content_type = 'markdown' "
+    "  AND source_path IS NOT NULL "
+    "ORDER BY id"
+)
+
+
+def backfill_source_rows(
+    conn: psycopg.Connection, *, commit: bool = True
+) -> BackfillReport:
+    """Set ``source_id`` to a "manual" sources row for legacy markdown docs.
+
+    Selection: ``content_type = 'markdown'`` AND ``source_id IS NULL`` AND
+    ``source_path IS NOT NULL``. For each match, upsert a ``sources`` row
+    with ``kind="manual"`` and ``external_id = source_path`` (so two docs
+    sharing a path collapse onto one source row), then point the document
+    at it.
+
+    Idempotent: a second run is a no-op because the WHERE clause filters
+    on ``source_id IS NULL``. The whole pass runs in a single transaction
+    — either every row gets fixed or none do. Set ``commit=False`` for a
+    preview that returns the candidate counts without writing.
+
+    Stdin-ingested rows (``source_path IS NULL``) are intentionally skipped.
+    Their kind cannot be inferred from the path alone — they would need
+    a per-source backfill (e.g. krisp re-ingest) instead.
+    """
+    rows = conn.execute(_BACKFILL_SQL).fetchall()
+    candidates = len(rows)
+
+    if not commit or candidates == 0:
+        return BackfillReport(
+            candidates=candidates,
+            sources_created=0,
+            documents_updated=0,
+            dry_run=not commit,
+        )
+
+    sources_before = conn.execute(
+        "SELECT count(*) FROM sources WHERE kind = 'manual'"
+    ).fetchone()
+    assert sources_before is not None  # COUNT(*) always returns one row
+    before = int(sources_before[0])
+
+    documents_updated = 0
+    with conn.transaction():
+        for doc_id, source_path in rows:
+            source_id = _upsert_source(
+                conn, kind="manual", external_id=source_path, metadata={}
+            )
+            # External_id is non-NULL for every row in this scan (WHERE
+            # clause filters source_path IS NOT NULL), so _upsert_source's
+            # "no external id and no metadata" early-return cannot fire.
+            assert source_id is not None
+            conn.execute(
+                "UPDATE documents SET source_id = %s WHERE id = %s",
+                (source_id, doc_id),
+            )
+            documents_updated += 1
+
+    sources_after = conn.execute(
+        "SELECT count(*) FROM sources WHERE kind = 'manual'"
+    ).fetchone()
+    assert sources_after is not None
+    sources_created = int(sources_after[0]) - before
+
+    return BackfillReport(
+        candidates=candidates,
+        sources_created=sources_created,
+        documents_updated=documents_updated,
+        dry_run=False,
+    )

brain/bin/__init__.py

@@ -0,0 +1 @@
+"""Python launchers for the brain-* console scripts."""

brain/bin/_launcher.py

@@ -0,0 +1,107 @@
+"""Shared shim-install drift-protection and execvpe helper for brain-* console scripts."""
+import contextlib
+import hashlib
+import os
+import sys
+import tempfile
+from collections.abc import Sequence
+from importlib.resources import files as resource_files
+from pathlib import Path
+from typing import NoReturn
+
+from ..config import _brain_home_root
+
+
+def _sha256(data: bytes) -> str:
+    """Return the lowercase hex SHA-256 digest of *data*."""
+    return hashlib.sha256(data).hexdigest()
+
+
+def ensure_shim(name: str, brain_home: Path) -> Path:
+    """Ensure $BRAIN_HOME/.shims/<name> is installed and up-to-date.
+
+    Copies from package data brain.templates.bin/<name>.sh, stripping the
+    .sh suffix at the installed path. Idempotent — safe to re-run.
+
+    Drift protection: if the installed shim's sha256 differs from the
+    package-data source, atomically replace via unique same-dir tmpfile
+    + os.replace. Concurrent-launcher races are safe because tempfile
+    names are unique.
+
+    Shims live under .shims/ (not bin/) to avoid colliding with the
+    dev-checkout bin/ wrappers when $BRAIN_HOME resolves to the repo
+    root. A pipx-installed user has $BRAIN_HOME = ~/.brain so .shims/
+    is fresh under that root; either way the shim location is
+    repo-bin-independent.
+
+    Returns the installed shim path.
+    """
+    # Read source bytes from package data (brain.templates.bin/<name>.sh).
+    src = resource_files("brain.templates.bin") / f"{name}.sh"
+    src_bytes: bytes = src.read_bytes()
+    src_hash = _sha256(src_bytes)
+
+    # Target path: <brain_home>/.shims/<name>  (no .sh suffix).
+    bin_dir = brain_home / ".shims"
+    bin_dir.mkdir(parents=True, exist_ok=True)
+    installed_path = bin_dir / name
+
+    # Check if the installed shim already exists and is up-to-date.
+    if installed_path.exists():
+        installed_bytes = installed_path.read_bytes()
+        if _sha256(installed_bytes) == src_hash:
+            # Already up-to-date — no-op (preserves mtime for dev workflows).
+            return installed_path
+        # Stale — atomically replace.
+        _atomic_write(src_bytes, installed_path, bin_dir)
+        print(f"shim updated: {name} (sha mismatch)", file=sys.stderr)
+    else:
+        # Fresh install.
+        _atomic_write(src_bytes, installed_path, bin_dir)
+        print(f"shim installed: {name}", file=sys.stderr)
+
+    return installed_path
+
+
+def _atomic_write(data: bytes, dest: Path, parent_dir: Path) -> None:
+    """Write *data* to *dest* atomically via a unique same-dir tmpfile.
+
+    Uses ``tempfile.NamedTemporaryFile(dir=parent_dir, delete=False)`` so the
+    temp file is on the same filesystem as *dest* — required for ``os.replace``
+    to be atomic. The random suffix from NamedTemporaryFile makes concurrent
+    launchers race-safe (each gets its own tmpfile; last ``os.replace`` wins
+    with the correct content).
+
+    On any failure after the tmpfile is created, the tmpfile is unlinked to
+    avoid leaving stale files in *parent_dir*.
+    """
+    tmp_path: str | None = None
+    try:
+        with tempfile.NamedTemporaryFile(dir=parent_dir, delete=False) as tf:
+            tmp_path = tf.name
+            tf.write(data)
+        Path(tmp_path).chmod(0o755)
+        os.replace(tmp_path, dest)
+        tmp_path = None  # os.replace succeeded — no cleanup needed.
+    except Exception:
+        if tmp_path is not None:
+            with contextlib.suppress(OSError):
+                os.unlink(tmp_path)
+        raise
+
+
+def exec_shim(name: str, args: Sequence[str]) -> NoReturn:
+    """Resolve $BRAIN_HOME, ensure the shim is installed/up-to-date, then
+    os.execvpe it with BRAIN_PY set in the env.
+
+    BRAIN_PY defaults to ``sys.executable`` (the interpreter running this
+    launcher — by construction the one that has ``brain`` importable).
+    An existing BRAIN_PY env var is preserved untouched so tests can stub
+    the interpreter (see ``tests/test_bin_scripts.py``) and advanced users
+    can pin a specific Python without editing the launcher.
+    """
+    brain_home = _brain_home_root()
+    shim = ensure_shim(name, brain_home)
+    env = dict(os.environ)
+    env.setdefault("BRAIN_PY", sys.executable)
+    os.execvpe(str(shim), [str(shim), *args], env)

brain/bin/down.py

@@ -0,0 +1,8 @@
+"""brain-down — stop the wiki + supervised watchers."""
+import sys
+
+from ._launcher import exec_shim
+
+
+def main() -> None:
+    exec_shim("brain-down", sys.argv[1:])

brain/bin/launchd.py

@@ -0,0 +1,268 @@
+"""Python launchd plist generator for brain-install-launchd / brain-uninstall-launchd.
+
+Renders the plist.j2 templates in brain.templates.launchd via manual str.replace
+with xml.sax.saxutils.escape so paths containing &/</>  can't produce invalid
+XML. Resolves PIPX_BIN_DIR via `pipx environment --value` with a ~/.local/bin
+fallback. Resolves the Python interpreter via sys.executable — the running brain
+process is by definition the right Python.
+
+Entry points (registered in pyproject.toml by T1.8):
+    brain-install-launchd  → install_main()
+    brain-uninstall-launchd → uninstall_main()
+"""
+
+import contextlib
+import os
+import subprocess
+import sys
+import tempfile
+import time
+import xml.etree.ElementTree as ET
+from importlib.resources import files as resource_files
+from pathlib import Path
+from xml.sax.saxutils import escape as xml_escape
+
+from ..config import _brain_home_root
+from ..errors import BrainError
+from ._launcher import ensure_shim
+
+# The launchd labels managed by brain. watcher/build are KeepAlive daemons;
+# brief (Plan 01) is a one-shot StartCalendarInterval job (07:00 daily). All are
+# installed + cleaned together — adding brief here means uninstall sweeps it too.
+_LABELS = ("com.brain.watcher", "com.brain.build", "com.brain.brief")
+
+
+class LaunchdError(BrainError):
+    """Raised when a launchd plist render or launchctl operation fails."""
+
+
+# ---------------------------------------------------------------------------
+# Public helpers (test-visible)
+# ---------------------------------------------------------------------------
+
+
+def resolve_pipx_bin_dir() -> Path:
+    """Return the pipx binary directory.
+
+    Calls ``pipx environment --value PIPX_BIN_DIR``; falls back to
+    ``~/.local/bin`` if pipx is not on PATH or the command fails.
+    """
+    try:
+        result = subprocess.run(
+            ["pipx", "environment", "--value", "PIPX_BIN_DIR"],
+            capture_output=True,
+            text=True,
+            check=True,
+            timeout=10,
+        )
+        raw = result.stdout.strip()
+        if raw:
+            return Path(raw)
+    except (FileNotFoundError, subprocess.CalledProcessError, subprocess.TimeoutExpired):
+        pass
+    return Path.home() / ".local" / "bin"
+
+
+def resolve_brain_py() -> Path:
+    """Resolve the Python interpreter that has ``brain`` importable.
+
+    Since this module is part of the brain package, ``sys.executable`` is
+    by definition the right Python — whether we are running inside a pipx venv,
+    a dev-checkout venv, or any other environment.
+    """
+    return Path(sys.executable)
+
+
+def render_plist(
+    label: str,
+    brain_home: Path,
+    vault_path: Path,
+    pipx_bin_dir: Path,
+    brain_py: Path,
+) -> str:
+    """Render the .plist.j2 template for *label* into a string.
+
+    XML-escapes every substituted value so the resulting document is always
+    valid XML, even when paths contain ``&``, ``<``, or ``>``.
+
+    Raises :class:`LaunchdError` if the rendered XML does not parse.
+    """
+    template_ref = resource_files("brain.templates.launchd") / f"{label}.plist.j2"
+    text: str = template_ref.read_text(encoding="utf-8")
+
+    substitutions: dict[str, str] = {
+        "brain_home": xml_escape(str(brain_home)),
+        "user_home": xml_escape(str(Path.home())),
+        "vault_path": xml_escape(str(vault_path)),
+        "pipx_bin_dir": xml_escape(str(pipx_bin_dir)),
+        "brain_py": xml_escape(str(brain_py)),
+    }
+    for key, value in substitutions.items():
+        text = text.replace("{{ " + key + " }}", value)
+
+    # Sanity-check: validate that the result is well-formed XML.
+    try:
+        ET.fromstring(text)
+    except ET.ParseError as exc:
+        raise LaunchdError(f"rendered plist for {label} is not valid XML: {exc}") from exc
+
+    return text
+
+
+def install_plists(
+    brain_home: Path,
+    launchd_dir: Path,
+    launchctl: str = "launchctl",
+    vault_path: Path | None = None,
+) -> None:
+    """Write both plists and load them into launchd. Idempotent.
+
+    Steps for each label:
+    1. Render the plist template.
+    2. Write to *launchd_dir*/<label>.plist atomically.
+    3. ``launchctl bootout`` (suppress errors — idempotent).
+    4. ``launchctl bootstrap`` (fail loudly on error).
+    """
+    launchd_dir.mkdir(parents=True, exist_ok=True)
+
+    # Install the foreground wrapper scripts the plists reference.
+    # ensure_shim() is idempotent — skips the write if the shim is current.
+    for wrapper in ("_brain-watcher-fg", "_brain-build-fg", "_brain-brief-fg"):
+        ensure_shim(wrapper, brain_home)
+
+    effective_vault_path = vault_path if vault_path is not None else Path.home() / "brain-vault"
+    pipx_bin_dir = resolve_pipx_bin_dir()
+    brain_py = resolve_brain_py()
+    uid: int = os.getuid()
+
+    for label in _LABELS:
+        plist_content = render_plist(
+            label,
+            brain_home=brain_home,
+            vault_path=effective_vault_path,
+            pipx_bin_dir=pipx_bin_dir,
+            brain_py=brain_py,
+        )
+        plist_path = launchd_dir / f"{label}.plist"
+
+        # Atomic write via same-dir tempfile + os.replace.
+        _atomic_write_text(plist_content, plist_path, launchd_dir)
+
+        domain = f"gui/{uid}/{label}"
+
+        # Bootout any prior incarnation — suppress errors when not loaded.
+        subprocess.run(
+            [launchctl, "bootout", domain],
+            check=False,
+            capture_output=True,
+        )
+
+        # Bootstrap — fail loudly if this step fails. Retry once on the
+        # "Bootstrap failed: 5: Input/output error" race that happens when
+        # the prior service hasn't fully torn down before the new one tries
+        # to load (observed on Apple Silicon under load).
+        for attempt in range(2):
+            result = subprocess.run(
+                [launchctl, "bootstrap", f"gui/{uid}", str(plist_path)],
+                check=False,
+                capture_output=True,
+                text=True,
+            )
+            if result.returncode == 0:
+                break
+            if attempt == 0:
+                time.sleep(1.0)
+                continue
+            stderr = result.stderr.strip()
+            raise LaunchdError(
+                f"launchctl bootstrap failed for {label}"
+                + (f": {stderr}" if stderr else "")
+            )
+
+
+def uninstall_plists(
+    launchd_dir: Path,
+    launchctl: str = "launchctl",
+) -> None:
+    """Bootout and remove both plists. Idempotent.
+
+    For each label:
+    1. Probe ``launchctl print gui/<uid>/<label>`` — if loaded, bootout.
+    2. Remove the plist file if it exists.
+    3. Print a "nothing to remove" message if neither action was taken.
+    """
+    uid: int = os.getuid()
+
+    for label in _LABELS:
+        plist_path = launchd_dir / f"{label}.plist"
+        domain = f"gui/{uid}/{label}"
+        did_something = False
+
+        # Check if the service is currently loaded.
+        probe = subprocess.run(
+            [launchctl, "print", domain],
+            check=False,
+            capture_output=True,
+        )
+        if probe.returncode == 0:
+            subprocess.run(
+                [launchctl, "bootout", domain],
+                check=False,
+                capture_output=True,
+            )
+            did_something = True
+
+        if plist_path.exists():
+            plist_path.unlink()
+            did_something = True
+
+        if not did_something:
+            print(f"{label}: nothing to remove")
+
+
+# ---------------------------------------------------------------------------
+# Entry points
+# ---------------------------------------------------------------------------
+
+
+def install_main() -> None:
+    """Entry point: brain-install-launchd."""
+    launchd_dir = Path(
+        os.environ.get("BRAIN_LAUNCHD_DIR") or Path.home() / "Library" / "LaunchAgents"
+    )
+    launchctl = os.environ.get("BRAIN_LAUNCHCTL") or "launchctl"
+    brain_home = _brain_home_root()
+    install_plists(brain_home, launchd_dir, launchctl)
+    print(f"🧠 brain LaunchAgents installed in {launchd_dir}")
+
+
+def uninstall_main() -> None:
+    """Entry point: brain-uninstall-launchd."""
+    launchd_dir = Path(
+        os.environ.get("BRAIN_LAUNCHD_DIR") or Path.home() / "Library" / "LaunchAgents"
+    )
+    launchctl = os.environ.get("BRAIN_LAUNCHCTL") or "launchctl"
+    uninstall_plists(launchd_dir, launchctl)
+
+
+# ---------------------------------------------------------------------------
+# Internal helpers
+# ---------------------------------------------------------------------------
+
+
+def _atomic_write_text(content: str, dest: Path, parent_dir: Path) -> None:
+    """Write *content* to *dest* atomically via a unique same-dir tempfile."""
+    tmp_path: str | None = None
+    try:
+        with tempfile.NamedTemporaryFile(
+            mode="w", encoding="utf-8", dir=parent_dir, delete=False, suffix=".tmp"
+        ) as tf:
+            tmp_path = tf.name
+            tf.write(content)
+        os.replace(tmp_path, dest)
+        tmp_path = None
+    except Exception:
+        if tmp_path is not None:
+            with contextlib.suppress(OSError):
+                os.unlink(tmp_path)
+        raise