brainlayer 1.5.45__tar.gz → 1.5.46__tar.gz

{brainlayer-1.5.45 → brainlayer-1.5.46}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: brainlayer
-Version: 1.5.45
+Version: 1.5.46
 Summary: Persistent memory MCP server for AI agents — 17 tools, semantic search, knowledge graph, on-device SQLite
 Project-URL: Homepage, https://brainlayer.etanheyman.com
 Project-URL: Repository, https://github.com/EtanHey/brainlayer

{brainlayer-1.5.45 → brainlayer-1.5.46}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "brainlayer"
-version = "1.5.45"
+version = "1.5.46"
 description = "Persistent memory MCP server for AI agents — 17 tools, semantic search, knowledge graph, on-device SQLite"
 license = {text = "Apache-2.0"}
 readme = "README.md"

{brainlayer-1.5.45 → brainlayer-1.5.46}/server.json

@@ -3,7 +3,7 @@
   "name": "io.github.etanhey/brainlayer",
   "title": "BrainLayer",
   "description": "Persistent memory for AI agents \u2014 local-first semantic search, knowledge graph, and 17 MCP tools with ToolAnnotations. One SQLite file, no cloud, no API keys.",
-  "version": "1.5.45",
+  "version": "1.5.46",
   "websiteUrl": "https://brainlayer.etanheyman.com",
   "repository": {
     "url": "https://github.com/EtanHey/brainlayer",
@@ -14,7 +14,7 @@
       "registryType": "pypi",
       "registryBaseUrl": "https://pypi.org",
       "identifier": "brainlayer",
-      "version": "1.5.45",
+      "version": "1.5.46",
       "runtimeHint": "pipx",
       "transport": {
         "type": "stdio"

{brainlayer-1.5.45 → brainlayer-1.5.46}/src/brainlayer/__init__.py

@@ -2,7 +2,7 @@
 
 from __future__ import annotations
 
-__version__ = "1.5.45"
+__version__ = "1.5.46"
 # Git sha the distributed package was built from; stamped by the release build, None in a source tree.
 # scripts/sprint_gate.py reads it in "keg" mode to prove the served package is the code under test.
 # No runtime `str | None` annotation: launchd/install.sh imports this under /usr/bin/python3 (3.9).

brainlayer-1.5.46/src/brainlayer/_build.py

@@ -0,0 +1 @@
+BUILD_SHA = "87ae86dbec7bc8f824bb742eed1ebac285d9724e"

{brainlayer-1.5.45 → brainlayer-1.5.46}/src/brainlayer/chunk_write.py

@@ -148,6 +148,62 @@ def _table_columns(conn: Any, table: str) -> set[str]:
     return {row[1] for row in rows}
 
 
+def scrub_chunk_values(values: Mapping[str, Any]) -> dict[str, Any]:
+    """Scrub stored text before deriving hashes, previews or merge inputs.
+
+    JSON fields are decoded/normalized before scanning so escaped credentials
+    and mapping keys cannot bypass the at-rest policy. Quarantine remains
+    metadata only. Already-scrubbed callers retain their findings.
+    """
+    from .pipeline.cloud_scrub import normalize_json_strings
+    from .pipeline.secret_scrub import merge_scrub_metadata, scrub_for_storage
+
+    clean = dict(values)
+    identity_fields = {"id", "brick_id", "conversation_id", "superseded_by", "aggregated_into"}
+    findings: dict[str, Any] = {}
+    body_findings: dict[str, Any] = {}
+    for key, value in clean.items():
+        structured = isinstance(value, (dict, list, tuple))
+        if structured:
+            value = json.dumps(value, ensure_ascii=False)
+        if not isinstance(value, str):
+            continue
+        if key != "content" and key not in identity_fields:
+            value = normalize_json_strings(value)
+        text, found = scrub_for_storage(value)
+        if key in identity_fields and text != value:
+            raise ValueError("chunk identifier requires redaction; refusing to change identity")
+        clean[key] = json.loads(text) if structured else text
+        # Quarantine count describes the body, not keys/derived placeholders.
+        findings = merge_scrub_metadata(
+            findings, {k: v for k, v in found.items() if k != "secret_scrub_quarantine_count"}
+        )
+        if key == "content":
+            body_findings = found
+    if findings or body_findings:
+        metadata = clean.get("metadata")
+        if metadata is None:
+            metadata = {}
+        if isinstance(metadata, str):
+            try:
+                metadata = json.loads(metadata)
+            except ValueError:
+                metadata = {"value": metadata}
+        if not isinstance(metadata, dict):
+            metadata = {"value": metadata}
+        if clean.get("content") == values.get("content") and (
+            "secret_scrub_quarantine_count" in metadata or "secret_scrub_redactions" in metadata
+        ):
+            body_findings.pop("secret_scrub_quarantine_count", None)
+        clean["metadata"] = merge_scrub_metadata(merge_scrub_metadata(metadata, findings), body_findings)
+    if clean.get("content") != values.get("content"):
+        clean["char_count"] = len(str(clean.get("content") or ""))
+        # Caller-supplied dedupe fields describe the pre-scrub text. Recompute.
+        for key in ("dedupe_hash", "simhash", *(f"simhash_band_{i}" for i in range(4))):
+            clean.pop(key, None)
+    return clean
+
+
 def prepare_canonical_insert(
     values: Mapping[str, Any],
     *,
@@ -158,6 +214,7 @@ def prepare_canonical_insert(
     """Fill required insert columns. Does not write."""
     from .timestamp_iso import normalize_timestamp
 
+    values = scrub_chunk_values(values)
     content = str(values.get("content") or "")
     if not content.strip():
         raise ValueError("content must be non-empty")
@@ -271,7 +328,12 @@ def insert_canonical_chunk(
     on_conflict: str = "ignore",
     now: datetime | None = None,
 ) -> dict[str, Any]:
-    """Insert one chunk row using the canonical column set."""
+    """Insert a scrubbed chunk; hashes and dedupe fields describe stored text.
+
+    Tokens of the same provider collapse to the same placeholder, so dedupe
+    treats otherwise identical scrubbed memories as the same content. Existing
+    unsanitized rows are not rewritten by this go-forward boundary.
+    """
     columns = _table_columns(conn, "chunks")
     row = prepare_canonical_insert(values, now=now, columns=columns)
     names = [key for key in row if key in columns]

{brainlayer-1.5.45 → brainlayer-1.5.46}/src/brainlayer/cli/__init__.py

@@ -1285,6 +1285,8 @@ def doctor_command(
             f"[bold]{status}[/] chunks={result.chunk_count} "
             f"recent_unvectored={result.recent_unvectored_chunks} queue={result.queue_count}"
         )
+        if result.cli_path_shadow and result.cli_path_shadow["state"] == "skipped":
+            rprint(f"CLI PATH check skipped: {result.cli_path_shadow['reason']}")
         for issue in result.issues:
             color = "red" if issue.severity == "fatal" else "yellow"
             rprint(f"[{color}]{issue.severity.upper()}[/] {issue.code}: {issue.message}")
@@ -2865,6 +2867,45 @@ def wal_checkpoint(
     report_job_alert("wal-checkpoint", None)
 
 
+@app.command("scrub-at-rest")
+def scrub_at_rest_command(
+    db_path: Path = typer.Option(..., "--db", help="Explicit database path; live apply requires guarded opt-in."),
+    providers: str = typer.Option("google_oauth", "--providers", help="Only google_oauth is supported."),
+    dry_run: bool = typer.Option(False, "--dry-run", help="Count matches without writing."),
+    batch_size: int = typer.Option(100, "--batch-size", min=1, max=1000),
+    allow_live_db: bool = typer.Option(
+        False, "--allow-live-db", help="Apply with maintenance safety gates and writer quiescence."
+    ),
+    expect_rows: int | None = typer.Option(
+        None, "--expect-rows", min=0, help="Required guarded apply total from the preceding dry run."
+    ),
+) -> None:
+    """Redact provider credentials on a copy or a guarded runtime DB; print counts only."""
+    from ..scrub_at_rest import scrub_at_rest
+
+    try:
+        result = scrub_at_rest(
+            db_path,
+            providers=providers,
+            dry_run=dry_run,
+            batch_size=batch_size,
+            allow_live_db=allow_live_db,
+            expect_rows=expect_rows,
+        )
+    except Exception as exc:
+        typer.echo(
+            json.dumps(
+                {
+                    "error": "at-rest scrub refused or failed",
+                    "error_type": type(exc).__name__,
+                    "reason": getattr(exc, "reason", "scrub-failed"),
+                }
+            )
+        )
+        raise typer.Exit(1) from None
+    typer.echo(json.dumps(result, sort_keys=True))
+
+
 @app.command("repair-fts")
 def repair_fts(
     db_path: Path | None = typer.Argument(None, help="Offline database copy; omitted means read-only FTS count check."),

{brainlayer-1.5.45 → brainlayer-1.5.46}/src/brainlayer/doctor.py

@@ -8,6 +8,9 @@ indexes.
 from __future__ import annotations
 
 import json
+import os
+import re
+import shlex
 import sqlite3
 import sys
 import time
@@ -250,11 +253,93 @@ class DoctorResult:
     hotlane_running: bool = False
     roundtrip_latency_seconds: float | None = None
     fts5_health: dict[str, Any] | None = None
+    cli_path_shadow: dict[str, Any] | None = None
 
     def to_dict(self) -> dict[str, Any]:
         return asdict(self)
 
 
+def _check_cli_path_shadow(result: DoctorResult) -> None:
+    """Keep an advisory PATH probe from aborting the other doctor checks."""
+    try:
+        _probe_cli_path_shadow(result)
+    except Exception as exc:
+        reason = f"{type(exc).__name__}: {exc}"
+        result.cli_path_shadow = {"state": "warning", "reason": reason}
+        result.issues.append(DoctorIssue("cli_path_shadow_check_failed", "warning", reason))
+
+
+def _probe_cli_path_shadow(result: DoctorResult) -> None:
+    prefix = Path(sys.prefix).resolve()
+    keg = next(
+        (p for p in (prefix, *prefix.parents) if p.parent.name == "brainlayer" and p.parent.parent.name == "Cellar"),
+        None,
+    )
+    result.cli_path_shadow = {"state": "skipped", "reason": "running interpreter is not a BrainLayer keg install"}
+    if keg is None:
+        return
+    paths: list[Path] = []
+    seen: set[Path] = set()
+    probe_failed = False
+    for directory in os.environ.get("PATH", "").split(os.pathsep):
+        candidate = Path(directory) / "brainlayer"
+        try:
+            if candidate.is_file() and os.access(candidate, os.X_OK):
+                resolved = candidate.resolve()
+                if resolved not in seen:
+                    paths.append(candidate.absolute())
+                    seen.add(resolved)
+        except OSError as exc:
+            probe_failed = True
+            result.issues.append(DoctorIssue("cli_path_shadow_probe_failed", "warning", f"Skipped {candidate}: {exc}"))
+    severity = "fatal" if paths and not paths[0].resolve().is_relative_to(keg) else "warning"
+    duplicates = [p for p in paths if not p.resolve().is_relative_to(keg)]
+    result.cli_path_shadow = {
+        "state": severity if duplicates or not paths or probe_failed else "pass",
+        "paths": [str(p) for p in paths],
+        "prefix": str(keg),
+    }
+    if not paths:
+        result.issues.append(DoctorIssue("cli_path_shadow", "warning", "No brainlayer executable on PATH"))
+    for path in duplicates:
+        severity = "fatal" if path == paths[0] else "warning"
+        kind, remedy = "plain script", "put the running BrainLayer keg bin directory first on PATH"
+        try:
+            with path.open() as stream:
+                shebang = stream.readline(4096).strip()
+            interpreter = Path(shebang[2:].split()[0]) if shebang.startswith("#!/") else None
+            if interpreter and interpreter.name.startswith("python"):
+                install = interpreter.parent.parent
+                metadata = list(install.glob("lib/python*/site-packages/brainlayer-*.dist-info/direct_url.json"))
+                version_match = re.search(r"python(\d+\.\d+)$", interpreter.resolve().name)
+                version = version_match.group(1) if version_match else None
+                user_metadata = []
+                if version:
+                    user_metadata = list(
+                        (Path.home() / "Library/Python" / version / "lib/python/site-packages").glob(
+                            "brainlayer-*.dist-info"
+                        )
+                    )
+                    user_metadata += list(
+                        (Path.home() / ".local/lib" / f"python{version}/site-packages").glob("brainlayer-*.dist-info")
+                    )
+                if any(json.loads(p.read_text()).get("dir_info", {}).get("editable") is True for p in metadata):
+                    kind = "editable install"
+                env = "PYTHONNOUSERSITE=1 " if user_metadata else ""
+                remedy = f"sudo {env}{shlex.quote(str(interpreter.parent / 'pip3'))} uninstall brainlayer"
+                if path.is_relative_to(Path.home() / ".local/bin") or path.is_relative_to(
+                    Path.home() / "Library/Python"
+                ):
+                    remedy = f"{shlex.quote(str(interpreter))} -m pip uninstall brainlayer"
+        except (OSError, ValueError, TypeError, AttributeError):
+            pass
+        result.issues.append(
+            DoctorIssue(
+                "cli_path_shadow", severity, f"{path}: {kind}; {remedy}", {"path": str(path), "remediation": remedy}
+            )
+        )
+
+
 def _ro_conn(db_path: Path) -> sqlite3.Connection:
     uri = f"file:{db_path.expanduser()}?mode=ro"
     conn = sqlite3.connect(uri, uri=True, timeout=5)
@@ -581,6 +666,8 @@ def run_doctor(
     def warning(code: str, message: str, **details: Any) -> None:
         result.issues.append(DoctorIssue(code, "warning", message, details))
 
+    _check_cli_path_shadow(result)
+
     db_config_error = get_brainlayer_db_config_error()
     if db_config_error is not None:
         fatal(

{brainlayer-1.5.45 → brainlayer-1.5.46}/src/brainlayer/hooks/indexer.py

@@ -181,6 +181,10 @@ class RealtimeIndexer:
         next_index = row[0] if row else 0
 
         try:
+            from ..pipeline.secret_scrub import scrub_secrets
+
+            compact_summary = scrub_secrets(compact_summary).text
+            trigger = scrub_secrets(trigger).text
             cursor = self._db.execute(
                 """INSERT INTO chapters (session_id, chapter_index, compact_summary, trigger)
                    VALUES (?, ?, ?, ?)""",

{brainlayer-1.5.45 → brainlayer-1.5.46}/src/brainlayer/maintenance.py

@@ -376,6 +376,8 @@ def quarantine_stale_queue_files(
 
 
 def _launchd_label(service: str) -> str:
+    if service == "fleet-watchdog":
+        return "com.etanhey.brainlayer-fleet-watchdog"
     return f"com.brainlayer.{service}"
 
 
@@ -437,6 +439,12 @@ def _verify_enrichment_template_flex_backend(repo_root_or_launchd_dir: Path) ->
 
 
 def _resume_service(repo_root: Path, service: str) -> None:
+    if service in {"brainbar", "brainbar-daemon", "fleet-watchdog"}:
+        # These already-installed jobs have no normal install.sh service option
+        # (or own a different namespace). Restore their existing configuration.
+        plist = Path.home() / "Library" / "LaunchAgents" / f"{_launchd_label(service)}.plist"
+        run_command(["launchctl", "bootstrap", f"gui/{os.getuid()}", str(plist)], check=True)
+        return
     launchd_dir = _launchd_dir_for_resume(repo_root)
     if service == "enrichment":
         _verify_enrichment_template_flex_backend(launchd_dir)
@@ -582,7 +590,7 @@ def _service_is_deliberately_paused(service: str) -> bool:
     payload, active, _stale = pause_sentinel_state(PAUSE_SENTINEL_PATH, datetime.now(UTC))
     if not active:
         return False
-    return pause_applies_to_label(payload, f"com.brainlayer.{service}")
+    return pause_applies_to_label(payload, _launchd_label(service))
 
 
 def _maintenance_keep_down_services() -> set[str]:

{brainlayer-1.5.45 → brainlayer-1.5.46}/src/brainlayer/pipeline/cloud_scrub.py

@@ -2,8 +2,9 @@
 
 Two directions, both fail closed:
 
-- ``scrub_for_cloud`` runs on every text sent to a cloud model. If scrubbing
-  raises, it raises ``CloudScrubError`` and the caller must not send.
+- ``scrub_for_cloud`` runs on every text sent to a cloud model. It
+  redacts provider secrets and quarantine spans. Scrub errors or unresolvable
+  quarantine records raise ``CloudScrubError`` and the caller must not send.
 - ``scrub_llm_output`` runs on every LLM output value before it is persisted.
   A cloud model copies tokens from its prompt into summaries and key facts,
   so output is scrubbed even when the input already was. If scrubbing raises,
@@ -32,7 +33,8 @@ class CloudScrubError(RuntimeError):
 
 def _scrub_text(text: str) -> str:
     try:
-        scrubbed = scrub_secrets(text).text
+        result = scrub_secrets(text)
+        scrubbed = result.text
     except Exception as exc:
         raise CloudScrubError(f"secret scrub failed ({type(exc).__name__}); refusing to pass text on") from None
     if not isinstance(scrubbed, str):
@@ -41,10 +43,47 @@ def _scrub_text(text: str) -> str:
 
 
 def scrub_for_cloud(text: str) -> str:
-    """Return ``text`` with secrets redacted, ready to send to a remote LLM."""
+    """Redact provider secrets and quarantine spans before a remote send.
+
+    A second pass finds quarantine offsets in the already-redacted text, so
+    provider replacements cannot shift them. Each offset must identify its value;
+    a mismatch redacts every occurrence, or fails closed if none can be found.
+    Storage retains its local-review quarantine policy; cloud prompts redact even
+    identifier-like false positives.
+    """
     if not isinstance(text, str):
         raise CloudScrubError(f"remote LLM payload must be text, got {type(text).__name__}")
-    return _scrub_text(text)
+    scrubbed = _scrub_text(text)
+    try:
+        quarantine = scrub_secrets(scrubbed).quarantine
+        spans: list[tuple[int, int]] = []
+        resolved_values: set[str] = set()
+        for token in quarantine:
+            if not token.value:
+                raise CloudScrubError("quarantine value is empty; refusing to send text")
+            if 0 <= token.start < token.end <= len(scrubbed) and scrubbed[token.start : token.end] == token.value:
+                spans.append((token.start, token.end))
+            elif token.value not in resolved_values:
+                start = scrubbed.find(token.value)
+                if start < 0:
+                    raise CloudScrubError("quarantine value not found; refusing to send text")
+                while start >= 0:
+                    end = start + len(token.value)
+                    spans.append((start, end))
+                    start = scrubbed.find(token.value, start + 1)
+                resolved_values.add(token.value)
+        parts: list[str] = []
+        cursor = 0
+        for start, end in sorted(spans):
+            if end <= cursor:
+                continue
+            if start >= cursor:
+                parts.extend((scrubbed[cursor:start], "[REDACTED:quarantine]"))
+            cursor = end
+        parts.append(scrubbed[cursor:])
+        return "".join(parts)
+    except Exception as exc:
+        raise CloudScrubError(f"secret scrub failed ({type(exc).__name__}); refusing to pass text on") from None
 
 
 def scrub_llm_output(value: T) -> T:

{brainlayer-1.5.45 → brainlayer-1.5.46}/src/brainlayer/pipeline/secret_scrub.py

@@ -60,6 +60,9 @@ _PROVIDER_PATTERNS = (
     ),
     _ProviderPattern("slack", re.compile(r"\bxox[baprs]-(?:[A-Za-z0-9]+-){1,}[A-Za-z0-9]{16,}\b")),
     _ProviderPattern("google", re.compile(r"\bAIza[A-Za-z0-9_-]{32,}\b")),
+    _ProviderPattern("google_oauth_access", re.compile(r"ya29\.[A-Za-z0-9_.-]{20,}(?![A-Za-z0-9_-])")),
+    _ProviderPattern("google_oauth_refresh", re.compile(r"1//[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])")),
+    _ProviderPattern("google_client_secret", re.compile(r"GOCSPX-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])")),
     _ProviderPattern("gitlab", re.compile(r"\bglpat-[A-Za-z0-9_-]{20,}\b")),
     _ProviderPattern("supabase", re.compile(r"\b(?:sbp_[A-Za-z0-9]{20,}|sb_secret_[A-Za-z0-9_-]{20,})\b")),
     _ProviderPattern("sendgrid", re.compile(r"\bSG\.[A-Za-z0-9_-]{16,}\.[A-Za-z0-9_-]{32,}\b")),
@@ -93,20 +96,28 @@ _HEX_RE = re.compile(r"\b[0-9a-fA-F]{16,}\b")
 _UUID_RE = re.compile(r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b")
 
 
-def scrub_secrets(text: str) -> SecretScrubResult:
-    """Redact labeled secrets before persistence while preserving join-key-like tokens."""
+def scrub_secrets(text: str, *, providers: frozenset[str] | None = None) -> SecretScrubResult:
+    """Redact secrets; an explicit provider set selects only those provider spans."""
+    if providers is not None and not providers <= {p.provider for p in _PROVIDER_PATTERNS}:
+        raise ValueError("unknown secret provider selector")
     if not text:
         return SecretScrubResult(text=text)
 
-    windows = list(_scan_windows(text))
+    # One-off provider selection must see real token boundaries, not window ends.
+    windows = [(0, len(text))] if providers is not None else list(_scan_windows(text))
     spans: list[SecretRedaction] = []
     for start, end in windows:
         spans.extend(_provider_redactions(text[start:end], offset=start))
-    for start, end in windows:
-        spans.extend(_assignment_redactions(text[start:end], spans, offset=start))
+    if providers is not None:
+        spans = [span for span in spans if span.provider in providers]
+    else:
+        for start, end in windows:
+            spans.extend(_assignment_redactions(text[start:end], spans, offset=start))
     spans = _without_overlaps(sorted(spans, key=lambda item: (item.start, item.end)))
 
     scrubbed = _apply_redactions(text, spans) if spans else text
+    if providers is not None:
+        return SecretScrubResult(text=scrubbed, redactions=spans)
     quarantine: list[QuarantinedToken] = []
     for start, end in windows:
         quarantine.extend(_quarantine_unlabeled_entropy(text[start:end], spans, offset=start))
@@ -198,8 +209,10 @@ def _quarantine_unlabeled_entropy(
     quarantined: list[QuarantinedToken] = []
     redaction_index = _SpanIndex(redactions)
     for match in _TOKEN_RE.finditer(text):
-        value = match.group(0).strip(".,;)")
-        start = offset + match.start()
+        raw_value = match.group(0)
+        value = raw_value.strip(".,;)")
+        leading_trim = len(raw_value) - len(raw_value.lstrip(".,;)"))
+        start = offset + match.start() + leading_trim
         end = start + len(value)
         if redaction_index.overlaps(start, end):
             continue