brainlayer 1.5.2__tar.gz → 1.5.3__tar.gz

{brainlayer-1.5.2 → brainlayer-1.5.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: brainlayer
-Version: 1.5.2
+Version: 1.5.3
 Summary: Persistent memory MCP server for AI agents — 13 tools, semantic search, knowledge graph, on-device SQLite
 Project-URL: Homepage, https://brainlayer.etanheyman.com
 Project-URL: Repository, https://github.com/EtanHey/brainlayer
@@ -23,7 +23,8 @@ Requires-Dist: abydos>=0.5.0
 Requires-Dist: apsw>=3.45.0
 Requires-Dist: google-api-python-client>=2.0.0
 Requires-Dist: google-auth>=2.0.0
-Requires-Dist: mcp>=1.0.0
+Requires-Dist: jsonschema>=4.20.0
+Requires-Dist: mcp<3.0.0,>=2.0.0
 Requires-Dist: numpy<3.0,>=1.22
 Requires-Dist: orjson>=3.9.0
 Requires-Dist: pydantic>=2.0.0

{brainlayer-1.5.2 → brainlayer-1.5.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "brainlayer"
-version = "1.5.2"
+version = "1.5.3"
 description = "Persistent memory MCP server for AI agents — 13 tools, semantic search, knowledge graph, on-device SQLite"
 license = {text = "Apache-2.0"}
 readme = "README.md"
@@ -16,7 +16,8 @@ dependencies = [
     "sentence-transformers>=2.2.0",
 
     # MCP server
-    "mcp>=1.0.0",
+    "mcp>=2.0.0,<3.0.0",
+    "jsonschema>=4.20.0",  # MCP v2 low-level handlers require explicit schema validation
 
     # CLI
     "typer>=0.9.0",

{brainlayer-1.5.2 → brainlayer-1.5.3}/scripts/hotlane_brainbar_daemon.py

@@ -15,6 +15,7 @@ import logging
 import os
 import signal
 import time
+from collections import OrderedDict
 from pathlib import Path
 from typing import Callable, NamedTuple
 
@@ -41,8 +42,88 @@ MAX_BACKLOG_BATCH = 16
 DEFAULT_HOTLANE_WRITE_BUSY_TIMEOUT_MS = 1000
 MAX_APSW_BUSY_TIMEOUT_MS = 2_147_483_647
 VECTOR_WRITE_YIELD_SECONDS = 0.005
+HOT_CANDIDATE_SCAN_LIMIT = 256
+HOT_CANDIDATE_HEAD_LIMIT = HOT_CANDIDATE_SCAN_LIMIT // 2
+MAX_HOT_CANDIDATE_RETRIES = 256
 _sleep = time.sleep
 
+HOT_CANDIDATE_SCAN_SQL = """
+    SELECT
+        c.id,
+        c.content,
+        c.source_file,
+        c.source,
+        c.archived_at,
+        c.superseded_by,
+        c.aggregated_into,
+        COALESCE(c.archived, 0),
+        COALESCE(c.status, 'active'),
+        r.id
+    FROM chunks c
+    LEFT JOIN chunk_vectors_rowids r ON r.id = c.id
+    ORDER BY c.created_at DESC
+    LIMIT ?
+"""
+
+HOT_CANDIDATE_ROWID_SCAN_SQL = """
+    SELECT
+        c.rowid,
+        c.id,
+        c.content,
+        c.source_file,
+        c.source,
+        c.archived_at,
+        c.superseded_by,
+        c.aggregated_into,
+        COALESCE(c.archived, 0),
+        COALESCE(c.status, 'active'),
+        r.id
+    FROM chunks c
+    LEFT JOIN chunk_vectors_rowids r ON r.id = c.id
+    ORDER BY c.rowid DESC
+    LIMIT ?
+"""
+
+HOT_CANDIDATE_ROWID_PAGE_SQL = """
+    SELECT
+        c.rowid,
+        c.id,
+        c.content,
+        c.source_file,
+        c.source,
+        c.archived_at,
+        c.superseded_by,
+        c.aggregated_into,
+        COALESCE(c.archived, 0),
+        COALESCE(c.status, 'active'),
+        r.id
+    FROM chunks c
+    LEFT JOIN chunk_vectors_rowids r ON r.id = c.id
+    WHERE c.rowid < ?
+    ORDER BY c.rowid DESC
+    LIMIT ?
+"""
+
+HOT_CANDIDATE_ROWID_FORWARD_SQL = """
+    SELECT
+        c.rowid,
+        c.id,
+        c.content,
+        c.source_file,
+        c.source,
+        c.archived_at,
+        c.superseded_by,
+        c.aggregated_into,
+        COALESCE(c.archived, 0),
+        COALESCE(c.status, 'active'),
+        r.id
+    FROM chunks c
+    LEFT JOIN chunk_vectors_rowids r ON r.id = c.id
+    WHERE c.rowid > ?
+    ORDER BY c.rowid ASC
+    LIMIT ?
+"""
+
 
 class CycleResult(NamedTuple):
     embedded: int = 0
@@ -64,6 +145,183 @@ class EmbeddedVector(NamedTuple):
     embedding: list[float]
 
 
+def _candidates_from_scanned_rows(
+    rows: list[tuple], *, limit: int, exclude_ids: set[str] | None = None
+) -> tuple[list[EmbedCandidate], int | None, int | None, bool]:
+    if limit <= 0:
+        return [], None, None, False
+    candidates: list[EmbedCandidate] = []
+    exclude_ids = exclude_ids or set()
+    first_candidate_rowid: int | None = None
+    last_inspected_rowid: int | None = None
+    for index, row in enumerate(rows):
+        (
+            rowid,
+            chunk_id,
+            content,
+            source_file,
+            source,
+            archived_at,
+            superseded_by,
+            aggregated_into,
+            archived,
+            status,
+            vector_id,
+        ) = row
+        last_inspected_rowid = int(rowid)
+        eligible = (
+            vector_id is None
+            and source_file == "brainbar-store"
+            and source == "mcp"
+            and content
+            and archived_at is None
+            and superseded_by is None
+            and aggregated_into is None
+            and not archived
+            and status == "active"
+        )
+        if eligible:
+            if first_candidate_rowid is None:
+                first_candidate_rowid = int(rowid)
+            if str(chunk_id) in exclude_ids:
+                continue
+            candidates.append(EmbedCandidate(str(chunk_id), str(content)))
+            if len(candidates) >= limit:
+                return candidates, first_candidate_rowid, last_inspected_rowid, index == len(rows) - 1
+    return candidates, first_candidate_rowid, last_inspected_rowid, True
+
+
+class HotCandidateScanner:
+    """Scan bounded rowid lanes while retaining returned candidates for retry."""
+
+    def __init__(self) -> None:
+        self._before_rowid: int | None = None
+        self._newest_seen_rowid: int | None = None
+        self._retries: OrderedDict[str, EmbedCandidate] = OrderedDict()
+        self._retry_turn = True
+        self._retry_buffer_full_logged = False
+
+    def _refresh_retries(self, cursor: apsw.Cursor) -> None:
+        if not self._retries:
+            return
+        placeholders = ",".join("?" for _ in self._retries)
+        rows = cursor.execute(
+            f"""
+            SELECT c.id, c.content
+            FROM chunks c
+            LEFT JOIN chunk_vectors_rowids r ON r.id = c.id
+            WHERE c.id IN ({placeholders})
+              AND r.id IS NULL
+              AND c.source_file = 'brainbar-store'
+              AND c.source = 'mcp'
+              AND c.content IS NOT NULL
+              AND c.content != ''
+              AND c.archived_at IS NULL
+              AND c.superseded_by IS NULL
+              AND c.aggregated_into IS NULL
+              AND COALESCE(c.archived, 0) = 0
+              AND COALESCE(c.status, 'active') = 'active'
+            """,
+            tuple(self._retries),
+        )
+        active = {str(chunk_id): EmbedCandidate(str(chunk_id), str(content)) for chunk_id, content in rows}
+        for chunk_id in list(self._retries):
+            if chunk_id not in active:
+                del self._retries[chunk_id]
+            else:
+                self._retries[chunk_id] = active[chunk_id]
+        if len(self._retries) < MAX_HOT_CANDIDATE_RETRIES:
+            self._retry_buffer_full_logged = False
+
+    def _take_retries(self, *, limit: int) -> list[EmbedCandidate]:
+        selected: list[EmbedCandidate] = []
+        for _ in range(min(limit, len(self._retries))):
+            chunk_id, candidate = self._retries.popitem(last=False)
+            self._retries[chunk_id] = candidate
+            selected.append(candidate)
+        return selected
+
+    def _retain(self, candidates: list[EmbedCandidate]) -> None:
+        for candidate in candidates:
+            if candidate.chunk_id in self._retries:
+                continue
+            if len(self._retries) >= MAX_HOT_CANDIDATE_RETRIES:
+                if not self._retry_buffer_full_logged:
+                    LOGGER.warning("hot candidate retry buffer full; pausing new retry retention")
+                    self._retry_buffer_full_logged = True
+                break
+            self._retries[candidate.chunk_id] = candidate
+
+    def __call__(self, store: VectorStore, *, limit: int) -> list[EmbedCandidate]:
+        cursor = store.conn.cursor()
+        self._refresh_retries(cursor)
+        if limit <= 0:
+            return []
+        if limit == 1 and self._retries:
+            retry_limit = 1 if self._retry_turn else 0
+            self._retry_turn = not self._retry_turn
+        else:
+            retry_limit = min(len(self._retries), max(1, limit // 2))
+        candidates = self._take_retries(limit=retry_limit)
+        retention_slots = MAX_HOT_CANDIDATE_RETRIES - len(self._retries)
+        scan_limit = min(limit - len(candidates), retention_slots)
+        if scan_limit <= 0:
+            return candidates
+        retry_ids = set(self._retries)
+        head_rows = list(cursor.execute(HOT_CANDIDATE_ROWID_SCAN_SQL, (HOT_CANDIDATE_HEAD_LIMIT,)))
+        head_candidates, _, last_inspected_rowid, _ = _candidates_from_scanned_rows(
+            head_rows,
+            limit=scan_limit,
+            exclude_ids=retry_ids,
+        )
+        candidates.extend(head_candidates)
+        retention_slots -= len(head_candidates)
+        if not head_rows:
+            self._retain(candidates)
+            return candidates
+        if self._newest_seen_rowid is None:
+            self._newest_seen_rowid = last_inspected_rowid
+        if self._before_rowid is None and last_inspected_rowid is not None:
+            self._before_rowid = last_inspected_rowid
+        if len(candidates) >= limit or retention_slots <= 0:
+            self._retain(candidates)
+            return candidates
+
+        page_limit = HOT_CANDIDATE_SCAN_LIMIT - HOT_CANDIDATE_HEAD_LIMIT
+        forward_rows = list(
+            cursor.execute(
+                HOT_CANDIDATE_ROWID_FORWARD_SQL,
+                (self._newest_seen_rowid, page_limit),
+            )
+        )
+        if forward_rows:
+            forward_candidates, _, last_inspected_rowid, _ = _candidates_from_scanned_rows(
+                forward_rows,
+                limit=min(limit - len(candidates), retention_slots),
+                exclude_ids=retry_ids | {candidate.chunk_id for candidate in candidates},
+            )
+            candidates.extend(forward_candidates)
+            retention_slots -= len(forward_candidates)
+            self._newest_seen_rowid = last_inspected_rowid
+        if len(candidates) >= limit or retention_slots <= 0 or self._before_rowid is None:
+            self._retain(candidates)
+            return candidates[:limit]
+
+        page_rows = list(cursor.execute(HOT_CANDIDATE_ROWID_PAGE_SQL, (self._before_rowid, page_limit)))
+        if page_rows:
+            page_candidates, _, last_inspected_rowid, fully_scanned = _candidates_from_scanned_rows(
+                page_rows,
+                limit=min(limit - len(candidates), retention_slots),
+                exclude_ids=retry_ids | {candidate.chunk_id for candidate in candidates},
+            )
+            candidates.extend(page_candidates)
+            self._before_rowid = last_inspected_rowid
+            if fully_scanned and len(page_rows) < page_limit:
+                self._before_rowid = None
+        self._retain(candidates)
+        return candidates[:limit]
+
+
 def _stop(_signum: int, _frame: object) -> None:
     global STOP
     STOP = True
@@ -95,26 +353,38 @@ def _candidate_chunk_ids(store: VectorStore, *, limit: int) -> list[str]:
 
 def _candidate_chunk_rows(store: VectorStore, *, limit: int) -> list[EmbedCandidate]:
     rows = store.conn.cursor().execute(
-        """
-        SELECT c.id, c.content
-        FROM chunks c
-        LEFT JOIN chunk_vectors_rowids r ON r.id = c.id
-        WHERE r.id IS NULL
-          AND c.source_file = 'brainbar-store'
-          AND c.source = 'mcp'
-          AND c.content IS NOT NULL
-          AND c.content != ''
-          AND c.archived_at IS NULL
-          AND c.superseded_by IS NULL
-          AND c.aggregated_into IS NULL
-          AND COALESCE(c.archived, 0) = 0
-          AND COALESCE(c.status, 'active') = 'active'
-        ORDER BY c.created_at DESC
-        LIMIT ?
-        """,
-        (limit,),
+        HOT_CANDIDATE_SCAN_SQL,
+        (HOT_CANDIDATE_SCAN_LIMIT,),
     )
-    return [EmbedCandidate(str(row[0]), str(row[1])) for row in rows]
+    candidates: list[EmbedCandidate] = []
+    for row in rows:
+        (
+            chunk_id,
+            content,
+            source_file,
+            source,
+            archived_at,
+            superseded_by,
+            aggregated_into,
+            archived,
+            status,
+            vector_id,
+        ) = row
+        if (
+            vector_id is None
+            and source_file == "brainbar-store"
+            and source == "mcp"
+            and content
+            and archived_at is None
+            and superseded_by is None
+            and aggregated_into is None
+            and not archived
+            and status == "active"
+        ):
+            candidates.append(EmbedCandidate(str(chunk_id), str(content)))
+            if len(candidates) >= limit:
+                break
+    return candidates
 
 
 def _pending_chunk_rows(store: VectorStore, *, limit: int) -> list[EmbedCandidate]:
@@ -419,9 +689,13 @@ def _queue_depth(queue_dir: Path) -> int:
         return 0
 
 
+def _is_enrichment_queue_file(path: Path) -> bool:
+    return path.name.startswith("enrichment-") or path.name.startswith("queue-enrichment")
+
+
 def _high_priority_queue_depth(queue_dir: Path) -> int:
     try:
-        return sum(1 for path in queue_dir.glob("*.jsonl") if not path.name.startswith("enrichment-"))
+        return sum(1 for path in queue_dir.glob("*.jsonl") if not _is_enrichment_queue_file(path))
     except OSError:
         return 0
 
@@ -505,6 +779,8 @@ def run(
     last_backlog = time_fn() - backlog_interval
     last_enrich = 0.0
     enrich_disabled = False
+    queue_backpressure_active = False
+    hot_candidate_scanner = HotCandidateScanner()
     cycles = 0
     backlog_batch = min(max(backlog_batch, 0), MAX_BACKLOG_BATCH)
     LOGGER.info("hotlane adapter started db=%s", db_path)
@@ -515,23 +791,23 @@ def run(
             now = time_fn()
             queue_has_backlog = queue_depth_fn(queue_dir) > 0
             queue_has_high_priority_backlog = queue_has_backlog and high_priority_queue_depth_fn(queue_dir) > 0
-            if queue_has_backlog:
-                if queue_has_high_priority_backlog:
+            if queue_has_high_priority_backlog:
+                if not queue_backpressure_active:
                     LOGGER.info("durable high-priority queue has backlog; yielding all hotlane writer work")
-                else:
-                    LOGGER.info("durable queue has backlog; yielding all hotlane writer work")
+                queue_backpressure_active = True
                 cycles += 1
                 sleep_fn(interval)
                 continue
-            else:
-                cycle_backlog_batch = (
-                    backlog_batch if backlog_batch > 0 and now - last_backlog >= backlog_interval else 0
-                )
-                cycle_enrich_limit = (
-                    enrich_limit
-                    if not enrich_disabled and enrich_limit > 0 and now - last_enrich >= enrich_interval
-                    else 0
-                )
+            queue_backpressure_active = False
+            cycle_backlog_batch = backlog_batch if backlog_batch > 0 and now - last_backlog >= backlog_interval else 0
+            cycle_enrich_limit = (
+                enrich_limit
+                if not queue_has_backlog
+                and not enrich_disabled
+                and enrich_limit > 0
+                and now - last_enrich >= enrich_interval
+                else 0
+            )
             if cycle_backlog_batch > 0:
                 last_backlog = now
             if cycle_enrich_limit > 0:
@@ -546,6 +822,7 @@ def run(
                     embed_batch_fn=embed_batch_fn,
                     enrich_limit=cycle_enrich_limit,
                     enrich_since_hours=enrich_since_hours,
+                    candidate_rows_fn=hot_candidate_scanner,
                 )
             else:
                 store = vector_store_cls(db_path)

{brainlayer-1.5.2 → brainlayer-1.5.3}/scripts/launchd/backup-daily.sh

@@ -3,8 +3,12 @@ set -euo pipefail
 
 export PATH="/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:$HOME/.local/bin"
 export PYTHONUNBUFFERED=1
-: "${BRAINLAYER_BACKUP_TIMEOUT_SECONDS:=300}"
+: "${BRAINLAYER_BACKUP_CLIENT_TIMEOUT_SECONDS:=0}"
+export BRAINLAYER_BACKUP_CLIENT_TIMEOUT_SECONDS
+: "${BRAINLAYER_BACKUP_TIMEOUT_SECONDS:=7200}"
 export BRAINLAYER_BACKUP_TIMEOUT_SECONDS
+: "${BRAINLAYER_BACKUP_ATTEMPT_MAX_AGE_SECONDS:=86400}"
+export BRAINLAYER_BACKUP_ATTEMPT_MAX_AGE_SECONDS
 : "${BRAINLAYER_BACKUP_LOG_PROVENANCE:=real}"
 export BRAINLAYER_BACKUP_LOG_PROVENANCE
 BRAINLAYER_DIR="${BRAINLAYER_DIR:-__BRAINLAYER_DIR_VALUE__}"

brainlayer-1.5.3/scripts/launchd/com.brainlayer.t3-ingest.plist

@@ -0,0 +1,53 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>Label</key>
+	<string>com.brainlayer.t3-ingest</string>
+	<key>ProgramArguments</key>
+	<array>
+		<string>__BRAINLAYER_ENV_RUN__</string>
+		<string>__BRAINLAYER_BIN__</string>
+		<string>ingest-t3</string>
+	</array>
+	<key>StartCalendarInterval</key>
+	<dict>
+		<key>Hour</key>
+		<integer>3</integer>
+		<key>Minute</key>
+		<integer>45</integer>
+	</dict>
+	<key>RunAtLoad</key>
+	<true/>
+	<key>StandardOutPath</key>
+	<string>__HOME__/Library/Logs/brainlayer/t3-ingest.out.log</string>
+	<key>StandardErrorPath</key>
+	<string>__HOME__/Library/Logs/brainlayer/t3-ingest.err.log</string>
+	<key>EnvironmentVariables</key>
+	<dict>
+		<key>PATH</key>
+		<string>/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:__HOME__/.local/bin</string>
+		<key>PYTHONUNBUFFERED</key>
+		<string>1</string>
+		<key>BRAINLAYER_REPO_ROOT</key>
+		<string>__BRAINLAYER_DIR__</string>
+		<key>BRAINLAYER_ENV_FILE</key>
+		<string>__BRAINLAYER_ENV_FILE__</string>
+		<key>BRAINLAYER_LAUNCHD_SERVICE</key>
+		<string>t3-ingest</string>
+	</dict>
+	<key>Nice</key>
+	<integer>10</integer>
+	<key>ProcessType</key>
+	<string>Background</string>
+	<key>ExitTimeOut</key>
+	<integer>1800</integer>
+	<key>LowPriorityIO</key>
+	<true/>
+	<key>SoftResourceLimits</key>
+	<dict>
+		<key>NumberOfFiles</key>
+		<integer>4096</integer>
+	</dict>
+</dict>
+</plist>

{brainlayer-1.5.2 → brainlayer-1.5.3}/scripts/launchd/com.brainlayer.wal-checkpoint.plist

@@ -11,11 +11,10 @@
 		<string>wal-checkpoint</string>
 		<string>--mode</string>
 		<string>TRUNCATE</string>
+		<string>--retry-busy</string>
 	</array>
 	<key>StartCalendarInterval</key>
 	<dict>
-		<key>Weekday</key>
-		<integer>0</integer>
 		<key>Hour</key>
 		<integer>4</integer>
 		<key>Minute</key>

{brainlayer-1.5.2 → brainlayer-1.5.3}/scripts/launchd/install.sh

@@ -20,6 +20,7 @@
 #   ./scripts/launchd/install.sh throughput-watchdog # Install watcher throughput watchdog only
 #   ./scripts/launchd/install.sh hotlane      # Install BrainBar hotlane embed/enrich daemon only
 #   ./scripts/launchd/install.sh p0-counter   # Install daily P0 longitudinal counter only
+#   ./scripts/launchd/install.sh t3-ingest    # Install T3 thread ingestion only
 #   ./scripts/launchd/install.sh remove       # Unload and remove all
 set -euo pipefail
 
@@ -661,13 +662,16 @@ case "${1:-all}" in
     p0-counter)
         install_plist p0-counter
         ;;
+    t3-ingest)
+        install_plist t3-ingest
+        ;;
     all)
         install_env_runner
         verify_config_file
         verify_gemini_env_file
         failures=0
         enrichment_ok=0
-        if ! install_many index drain watch hotlane-brainbar; then
+        if ! install_many index t3-ingest drain watch hotlane-brainbar; then
             failures=1
         fi
         if install_plist enrichment; then
@@ -707,6 +711,7 @@ case "${1:-all}" in
         ;;
     remove)
         remove_plist index
+        remove_plist t3-ingest 2>/dev/null || true
         remove_plist enrich 2>/dev/null || true
         remove_plist enrichment 2>/dev/null || true
         remove_plist watch 2>/dev/null || true
@@ -730,7 +735,7 @@ case "${1:-all}" in
         rm -f "$HOTLANE_BRAINBAR_DST"
         ;;
     *)
-        echo "Usage: $0 [index|watch|enrich|enrichment|decay|drain|hotlane|repair-fts|load [name]|unload [name]|checkpoint|backup|jsonl-backup|maintenance|maintenance-nightly|maintenance-weekly|health-check|tier0-watchdog|throughput-watchdog|p0-counter|all|remove]"
+        echo "Usage: $0 [index|t3-ingest|watch|enrich|enrichment|decay|drain|hotlane|repair-fts|load [name]|unload [name]|checkpoint|backup|jsonl-backup|maintenance|maintenance-nightly|maintenance-weekly|health-check|tier0-watchdog|throughput-watchdog|p0-counter|all|remove]"
         exit 1
         ;;
 esac