brainlayer 1.5.45__tar.gz → 1.5.47__tar.gz

{brainlayer-1.5.45 → brainlayer-1.5.47}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: brainlayer
-Version: 1.5.45
+Version: 1.5.47
 Summary: Persistent memory MCP server for AI agents — 17 tools, semantic search, knowledge graph, on-device SQLite
 Project-URL: Homepage, https://brainlayer.etanheyman.com
 Project-URL: Repository, https://github.com/EtanHey/brainlayer

{brainlayer-1.5.45 → brainlayer-1.5.47}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "brainlayer"
-version = "1.5.45"
+version = "1.5.47"
 description = "Persistent memory MCP server for AI agents — 17 tools, semantic search, knowledge graph, on-device SQLite"
 license = {text = "Apache-2.0"}
 readme = "README.md"

{brainlayer-1.5.45 → brainlayer-1.5.47}/server.json

@@ -3,7 +3,7 @@
   "name": "io.github.etanhey/brainlayer",
   "title": "BrainLayer",
   "description": "Persistent memory for AI agents \u2014 local-first semantic search, knowledge graph, and 17 MCP tools with ToolAnnotations. One SQLite file, no cloud, no API keys.",
-  "version": "1.5.45",
+  "version": "1.5.47",
   "websiteUrl": "https://brainlayer.etanheyman.com",
   "repository": {
     "url": "https://github.com/EtanHey/brainlayer",
@@ -14,7 +14,7 @@
       "registryType": "pypi",
       "registryBaseUrl": "https://pypi.org",
       "identifier": "brainlayer",
-      "version": "1.5.45",
+      "version": "1.5.47",
       "runtimeHint": "pipx",
       "transport": {
         "type": "stdio"

{brainlayer-1.5.45 → brainlayer-1.5.47}/src/brainlayer/__init__.py

@@ -2,7 +2,7 @@
 
 from __future__ import annotations
 
-__version__ = "1.5.45"
+__version__ = "1.5.47"
 # Git sha the distributed package was built from; stamped by the release build, None in a source tree.
 # scripts/sprint_gate.py reads it in "keg" mode to prove the served package is the code under test.
 # No runtime `str | None` annotation: launchd/install.sh imports this under /usr/bin/python3 (3.9).

brainlayer-1.5.47/src/brainlayer/_build.py

@@ -0,0 +1 @@
+BUILD_SHA = "38521b5296eddaaa7d2cdc722119f2622f3456b2"

{brainlayer-1.5.45 → brainlayer-1.5.47}/src/brainlayer/chunk_write.py

@@ -148,6 +148,62 @@ def _table_columns(conn: Any, table: str) -> set[str]:
     return {row[1] for row in rows}
 
 
+def scrub_chunk_values(values: Mapping[str, Any]) -> dict[str, Any]:
+    """Scrub stored text before deriving hashes, previews or merge inputs.
+
+    JSON fields are decoded/normalized before scanning so escaped credentials
+    and mapping keys cannot bypass the at-rest policy. Quarantine remains
+    metadata only. Already-scrubbed callers retain their findings.
+    """
+    from .pipeline.cloud_scrub import normalize_json_strings
+    from .pipeline.secret_scrub import merge_scrub_metadata, scrub_for_storage
+
+    clean = dict(values)
+    identity_fields = {"id", "brick_id", "conversation_id", "superseded_by", "aggregated_into"}
+    findings: dict[str, Any] = {}
+    body_findings: dict[str, Any] = {}
+    for key, value in clean.items():
+        structured = isinstance(value, (dict, list, tuple))
+        if structured:
+            value = json.dumps(value, ensure_ascii=False)
+        if not isinstance(value, str):
+            continue
+        if key != "content" and key not in identity_fields:
+            value = normalize_json_strings(value)
+        text, found = scrub_for_storage(value)
+        if key in identity_fields and text != value:
+            raise ValueError("chunk identifier requires redaction; refusing to change identity")
+        clean[key] = json.loads(text) if structured else text
+        # Quarantine count describes the body, not keys/derived placeholders.
+        findings = merge_scrub_metadata(
+            findings, {k: v for k, v in found.items() if k != "secret_scrub_quarantine_count"}
+        )
+        if key == "content":
+            body_findings = found
+    if findings or body_findings:
+        metadata = clean.get("metadata")
+        if metadata is None:
+            metadata = {}
+        if isinstance(metadata, str):
+            try:
+                metadata = json.loads(metadata)
+            except ValueError:
+                metadata = {"value": metadata}
+        if not isinstance(metadata, dict):
+            metadata = {"value": metadata}
+        if clean.get("content") == values.get("content") and (
+            "secret_scrub_quarantine_count" in metadata or "secret_scrub_redactions" in metadata
+        ):
+            body_findings.pop("secret_scrub_quarantine_count", None)
+        clean["metadata"] = merge_scrub_metadata(merge_scrub_metadata(metadata, findings), body_findings)
+    if clean.get("content") != values.get("content"):
+        clean["char_count"] = len(str(clean.get("content") or ""))
+        # Caller-supplied dedupe fields describe the pre-scrub text. Recompute.
+        for key in ("dedupe_hash", "simhash", *(f"simhash_band_{i}" for i in range(4))):
+            clean.pop(key, None)
+    return clean
+
+
 def prepare_canonical_insert(
     values: Mapping[str, Any],
     *,
@@ -158,6 +214,7 @@ def prepare_canonical_insert(
     """Fill required insert columns. Does not write."""
     from .timestamp_iso import normalize_timestamp
 
+    values = scrub_chunk_values(values)
     content = str(values.get("content") or "")
     if not content.strip():
         raise ValueError("content must be non-empty")
@@ -271,7 +328,12 @@ def insert_canonical_chunk(
     on_conflict: str = "ignore",
     now: datetime | None = None,
 ) -> dict[str, Any]:
-    """Insert one chunk row using the canonical column set."""
+    """Insert a scrubbed chunk; hashes and dedupe fields describe stored text.
+
+    Tokens of the same provider collapse to the same placeholder, so dedupe
+    treats otherwise identical scrubbed memories as the same content. Existing
+    unsanitized rows are not rewritten by this go-forward boundary.
+    """
     columns = _table_columns(conn, "chunks")
     row = prepare_canonical_insert(values, now=now, columns=columns)
     names = [key for key in row if key in columns]

{brainlayer-1.5.45 → brainlayer-1.5.47}/src/brainlayer/cli/__init__.py

@@ -1285,6 +1285,8 @@ def doctor_command(
             f"[bold]{status}[/] chunks={result.chunk_count} "
             f"recent_unvectored={result.recent_unvectored_chunks} queue={result.queue_count}"
         )
+        if result.cli_path_shadow and result.cli_path_shadow["state"] == "skipped":
+            rprint(f"CLI PATH check skipped: {result.cli_path_shadow['reason']}")
         for issue in result.issues:
             color = "red" if issue.severity == "fatal" else "yellow"
             rprint(f"[{color}]{issue.severity.upper()}[/] {issue.code}: {issue.message}")
@@ -2865,6 +2867,50 @@ def wal_checkpoint(
     report_job_alert("wal-checkpoint", None)
 
 
+@app.command("scrub-at-rest")
+def scrub_at_rest_command(
+    db_path: Path = typer.Option(..., "--db", help="Explicit database path; live apply requires guarded opt-in."),
+    providers: str = typer.Option(
+        "google_oauth",
+        "--providers",
+        help="Provider mode: google_oauth, context7, or exa_labeled (EXA key labels only).",
+    ),
+    dry_run: bool = typer.Option(False, "--dry-run", help="Count matches without writing."),
+    batch_size: int = typer.Option(100, "--batch-size", min=1, max=1000),
+    allow_live_db: bool = typer.Option(
+        False, "--allow-live-db", help="Apply with maintenance safety gates and writer quiescence."
+    ),
+    expect_rows: int | None = typer.Option(
+        None, "--expect-rows", min=0, help="Required guarded apply total from the preceding dry run."
+    ),
+) -> None:
+    """Redact provider credentials on a copy or a guarded runtime DB; print counts only."""
+    from ..scrub_at_rest import scrub_at_rest
+
+    try:
+        result = scrub_at_rest(
+            db_path,
+            providers=providers,
+            dry_run=dry_run,
+            batch_size=batch_size,
+            allow_live_db=allow_live_db,
+            expect_rows=expect_rows,
+        )
+    except Exception as exc:
+        typer.echo(
+            json.dumps(
+                {
+                    "error": "at-rest scrub refused or failed",
+                    "error_type": type(exc).__name__,
+                    "reason": getattr(exc, "reason", "scrub-failed"),
+                    "detail": getattr(exc, "detail", None),
+                }
+            )
+        )
+        raise typer.Exit(1) from None
+    typer.echo(json.dumps(result, sort_keys=True))
+
+
 @app.command("repair-fts")
 def repair_fts(
     db_path: Path | None = typer.Argument(None, help="Offline database copy; omitted means read-only FTS count check."),

{brainlayer-1.5.45 → brainlayer-1.5.47}/src/brainlayer/doctor.py

@@ -8,6 +8,9 @@ indexes.
 from __future__ import annotations
 
 import json
+import os
+import re
+import shlex
 import sqlite3
 import sys
 import time
@@ -250,11 +253,93 @@ class DoctorResult:
     hotlane_running: bool = False
     roundtrip_latency_seconds: float | None = None
     fts5_health: dict[str, Any] | None = None
+    cli_path_shadow: dict[str, Any] | None = None
 
     def to_dict(self) -> dict[str, Any]:
         return asdict(self)
 
 
+def _check_cli_path_shadow(result: DoctorResult) -> None:
+    """Keep an advisory PATH probe from aborting the other doctor checks."""
+    try:
+        _probe_cli_path_shadow(result)
+    except Exception as exc:
+        reason = f"{type(exc).__name__}: {exc}"
+        result.cli_path_shadow = {"state": "warning", "reason": reason}
+        result.issues.append(DoctorIssue("cli_path_shadow_check_failed", "warning", reason))
+
+
+def _probe_cli_path_shadow(result: DoctorResult) -> None:
+    prefix = Path(sys.prefix).resolve()
+    keg = next(
+        (p for p in (prefix, *prefix.parents) if p.parent.name == "brainlayer" and p.parent.parent.name == "Cellar"),
+        None,
+    )
+    result.cli_path_shadow = {"state": "skipped", "reason": "running interpreter is not a BrainLayer keg install"}
+    if keg is None:
+        return
+    paths: list[Path] = []
+    seen: set[Path] = set()
+    probe_failed = False
+    for directory in os.environ.get("PATH", "").split(os.pathsep):
+        candidate = Path(directory) / "brainlayer"
+        try:
+            if candidate.is_file() and os.access(candidate, os.X_OK):
+                resolved = candidate.resolve()
+                if resolved not in seen:
+                    paths.append(candidate.absolute())
+                    seen.add(resolved)
+        except OSError as exc:
+            probe_failed = True
+            result.issues.append(DoctorIssue("cli_path_shadow_probe_failed", "warning", f"Skipped {candidate}: {exc}"))
+    severity = "fatal" if paths and not paths[0].resolve().is_relative_to(keg) else "warning"
+    duplicates = [p for p in paths if not p.resolve().is_relative_to(keg)]
+    result.cli_path_shadow = {
+        "state": severity if duplicates or not paths or probe_failed else "pass",
+        "paths": [str(p) for p in paths],
+        "prefix": str(keg),
+    }
+    if not paths:
+        result.issues.append(DoctorIssue("cli_path_shadow", "warning", "No brainlayer executable on PATH"))
+    for path in duplicates:
+        severity = "fatal" if path == paths[0] else "warning"
+        kind, remedy = "plain script", "put the running BrainLayer keg bin directory first on PATH"
+        try:
+            with path.open() as stream:
+                shebang = stream.readline(4096).strip()
+            interpreter = Path(shebang[2:].split()[0]) if shebang.startswith("#!/") else None
+            if interpreter and interpreter.name.startswith("python"):
+                install = interpreter.parent.parent
+                metadata = list(install.glob("lib/python*/site-packages/brainlayer-*.dist-info/direct_url.json"))
+                version_match = re.search(r"python(\d+\.\d+)$", interpreter.resolve().name)
+                version = version_match.group(1) if version_match else None
+                user_metadata = []
+                if version:
+                    user_metadata = list(
+                        (Path.home() / "Library/Python" / version / "lib/python/site-packages").glob(
+                            "brainlayer-*.dist-info"
+                        )
+                    )
+                    user_metadata += list(
+                        (Path.home() / ".local/lib" / f"python{version}/site-packages").glob("brainlayer-*.dist-info")
+                    )
+                if any(json.loads(p.read_text()).get("dir_info", {}).get("editable") is True for p in metadata):
+                    kind = "editable install"
+                env = "PYTHONNOUSERSITE=1 " if user_metadata else ""
+                remedy = f"sudo {env}{shlex.quote(str(interpreter.parent / 'pip3'))} uninstall brainlayer"
+                if path.is_relative_to(Path.home() / ".local/bin") or path.is_relative_to(
+                    Path.home() / "Library/Python"
+                ):
+                    remedy = f"{shlex.quote(str(interpreter))} -m pip uninstall brainlayer"
+        except (OSError, ValueError, TypeError, AttributeError):
+            pass
+        result.issues.append(
+            DoctorIssue(
+                "cli_path_shadow", severity, f"{path}: {kind}; {remedy}", {"path": str(path), "remediation": remedy}
+            )
+        )
+
+
 def _ro_conn(db_path: Path) -> sqlite3.Connection:
     uri = f"file:{db_path.expanduser()}?mode=ro"
     conn = sqlite3.connect(uri, uri=True, timeout=5)
@@ -581,6 +666,8 @@ def run_doctor(
     def warning(code: str, message: str, **details: Any) -> None:
         result.issues.append(DoctorIssue(code, "warning", message, details))
 
+    _check_cli_path_shadow(result)
+
     db_config_error = get_brainlayer_db_config_error()
     if db_config_error is not None:
         fatal(

{brainlayer-1.5.45 → brainlayer-1.5.47}/src/brainlayer/hooks/indexer.py

@@ -181,6 +181,10 @@ class RealtimeIndexer:
         next_index = row[0] if row else 0
 
         try:
+            from ..pipeline.secret_scrub import scrub_secrets
+
+            compact_summary = scrub_secrets(compact_summary).text
+            trigger = scrub_secrets(trigger).text
             cursor = self._db.execute(
                 """INSERT INTO chapters (session_id, chapter_index, compact_summary, trigger)
                    VALUES (?, ?, ?, ?)""",

{brainlayer-1.5.45 → brainlayer-1.5.47}/src/brainlayer/jsonl_backup.py

@@ -50,6 +50,8 @@ import traceback
 import uuid
 from dataclasses import dataclass
 from pathlib import Path
+from time import monotonic as _monotonic
+from time import sleep as _sleep
 from typing import Any
 
 from . import backup_daily
@@ -190,7 +192,7 @@ def _icloud_deadline_for_archive(
     """Set an archive-sized budget when the iCloud copy operation begins."""
     archive_size = Path(archive_path).expanduser().stat().st_size
     configured_timeout = _configured_icloud_timeout_seconds() if timeout_seconds is None else timeout_seconds
-    return time.monotonic() + max(configured_timeout, archive_size / ICLOUD_MIN_BYTES_PER_SECOND)
+    return _monotonic() + max(configured_timeout, archive_size / ICLOUD_MIN_BYTES_PER_SECOND)
 
 
 def _load_state(path: Path) -> dict[str, Any]:
@@ -454,7 +456,7 @@ class ICloudProbeError(RuntimeError):
 
 
 def _check_icloud_deadline(deadline: float | None, phase: str) -> None:
-    if deadline is not None and time.monotonic() >= deadline:
+    if deadline is not None and _monotonic() >= deadline:
         raise ICloudDeadlineExceeded(f"iCloud operation deadline exceeded during {phase}")
 
 
@@ -663,7 +665,7 @@ def copy_archive_to_icloud(
     )
 
     def remaining_seconds() -> float:
-        return max(deadline - time.monotonic(), 0.001)
+        return max(deadline - _monotonic(), 0.001)
 
     icloud_dir.mkdir(parents=True, exist_ok=True)
     _check_icloud_deadline(deadline, "preparing the iCloud directory")
@@ -738,14 +740,14 @@ def copy_archive_to_icloud(
                     result = receipt(reused=True)
                     _trash_stale_unverified_icloud_items(icloud_dir, logical_sha256)
                     return result
-                if time.monotonic() >= deadline:
+                if _monotonic() >= deadline:
                     if state.get("is_uploading") is True and destination.is_file():
                         return pending_receipt()
                     raise ICloudDeadlineExceeded(
                         "existing iCloud copy was not uploaded and materialized before its deadline: "
                         f"path={destination} state={state!r}"
                     )
-                time.sleep(min(poll_interval_seconds, remaining_seconds()))
+                _sleep(min(poll_interval_seconds, remaining_seconds()))
                 status_path = placeholder if not destination.exists() and placeholder.exists() else destination
                 state = _icloud_item_state(
                     status_path,
@@ -767,7 +769,7 @@ def copy_archive_to_icloud(
                 raise
             _quarantine_unverified_icloud_item(destination)
             _quarantine_unverified_icloud_item(placeholder)
-            if time.monotonic() >= deadline:
+            if _monotonic() >= deadline:
                 raise
 
     temp_path = icloud_dir / f".{destination.name}.{os.getpid()}.partial"
@@ -800,10 +802,10 @@ def copy_archive_to_icloud(
     # The local copy/hash budget is complete. Network status gets a fresh phase
     # budget, so an expired disk-scaled deadline cannot become a 0.001-second
     # osascript timeout on the upload probe.
-    network_deadline = time.monotonic() + configured_timeout
+    network_deadline = _monotonic() + configured_timeout
 
     def network_remaining_seconds() -> float:
-        return max(network_deadline - time.monotonic(), 0.001)
+        return max(network_deadline - _monotonic(), 0.001)
 
     verified = False
     pending = False
@@ -841,10 +843,10 @@ def copy_archive_to_icloud(
                 )
                 _trash_stale_unverified_icloud_items(icloud_dir, logical_sha256)
                 return result
-            if time.monotonic() >= deadline and state.get("is_uploading") is True and destination.is_file():
+            if _monotonic() >= deadline and state.get("is_uploading") is True and destination.is_file():
                 pending = True
                 return pending_receipt()
-            if time.monotonic() >= network_deadline:
+            if _monotonic() >= network_deadline:
                 if state.get("is_uploading") is True and destination.is_file():
                     pending = True
                     return pending_receipt()
@@ -854,7 +856,7 @@ def copy_archive_to_icloud(
                     "iCloud copy was not uploaded and materialized before its deadline: "
                     f"path={destination} materialization={materialization} state={state!r}"
                 )
-            time.sleep(min(poll_interval_seconds, network_remaining_seconds()))
+            _sleep(min(poll_interval_seconds, network_remaining_seconds()))
             placeholder = destination.with_name(f".{destination.name}.icloud")
             status_path = placeholder if not destination.exists() and placeholder.exists() else destination
             state = _icloud_item_state(status_path, request_download=True, timeout_seconds=network_remaining_seconds())
@@ -1042,7 +1044,7 @@ def _icloud_inventory_is_verified(
         return False
 
     configured_timeout = _configured_icloud_timeout_seconds() if timeout_seconds is None else timeout_seconds
-    deadline = deadline if deadline is not None else time.monotonic() + configured_timeout
+    deadline = deadline if deadline is not None else _monotonic() + configured_timeout
     for archive_name in sorted(referenced_by_sources):
         try:
             _check_icloud_deadline(deadline, f"validating iCloud inventory archive {archive_name}")
@@ -1086,7 +1088,7 @@ def _icloud_inventory_is_verified(
             archive_valid = True
             while True:
                 _check_icloud_deadline(deadline, f"probing iCloud inventory archive {archive_name}")
-                remaining = max(deadline - time.monotonic(), 0.001)
+                remaining = max(deadline - _monotonic(), 0.001)
                 status_path = placeholder if not destination.exists() and placeholder.exists() else destination
                 item_state = _icloud_item_state(
                     status_path,
@@ -1117,11 +1119,11 @@ def _icloud_inventory_is_verified(
                     elif validated_sources is not None:
                         validated_sources.update(referenced_by_sources[archive_name])
                     break
-                if time.monotonic() >= deadline:
+                if _monotonic() >= deadline:
                     raise ICloudDeadlineExceeded(
                         f"iCloud operation deadline exceeded while materializing inventory archive {archive_name}"
                     )
-                time.sleep(min(poll_interval_seconds, remaining))
+                _sleep(min(poll_interval_seconds, remaining))
             if not archive_valid:
                 complete = False
         except (OSError, RuntimeError, subprocess.TimeoutExpired) as exc: