opencode-semantic-memory 0.8.2__tar.gz → 0.8.3__tar.gz

{opencode_semantic_memory-0.8.2 → opencode_semantic_memory-0.8.3}/CHANGELOG.md

@@ -7,6 +7,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.8.3] - 2026-06-03
+
+### Fixed
+
+- **Concurrent subagent MCP connections** - Background tasks that spawned multiple
+  subagents at once could fail silently with `MCP error -32000: Connection closed`
+  when their `mcp-remote` clients all performed their startup handshake against the
+  shared HTTP server simultaneously. Two fixes:
+  - `scripts/memory-mcp.sh` now retries `mcp-remote` with exponential backoff
+    (up to 5 attempts) so a transient startup-time connection drop recovers instead
+    of leaving the subagent without memory access.
+  - The HTTP server's rate limiter now exempts loopback addresses (all local
+    sessions are trusted) and keys non-local clients by the `X-OpenCode-Session`
+    header, so concurrent sessions no longer share a single localhost bucket whose
+    limit a startup burst could trip.
+
 ## [0.8.0] - 2026-06-02
 
 ### Added

{opencode_semantic_memory-0.8.2 → opencode_semantic_memory-0.8.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: opencode-semantic-memory
-Version: 0.8.2
+Version: 0.8.3
 Summary: Persistent semantic memory system for OpenCode sessions
 Author-email: Gregory Havenga <ghavenga@gitlab.com>
 License-Expression: MIT

{opencode_semantic_memory-0.8.2 → opencode_semantic_memory-0.8.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "opencode-semantic-memory"
-version = "0.8.2"
+version = "0.8.3"
 description = "Persistent semantic memory system for OpenCode sessions"
 readme = "README.md"
 license = "MIT"
@@ -52,6 +52,8 @@ opencode-memory = "opencode_memory.cli:main"
 opencode-memory-server = "opencode_memory.http_server:main"
 opencode-memory-daemon = "opencode_memory.daemon:main"
 opencode-memory-worker = "opencode_memory.background_worker:main"
+# New job-based worker (Sidekiq-like)
+opencode-memory-job-worker = "opencode_memory.jobs.worker:main"
 # Bulk ingestion commands
 opencode-memory-bulk-ingest = "opencode_memory.bulk_ingest.cli:main"
 opencode-memory-bulk-ingest-daemon = "opencode_memory.bulk_ingest.daemon:main"

opencode_semantic_memory-0.8.3/scripts/memory-mcp.sh

@@ -0,0 +1,65 @@
+#!/bin/bash
+# Wrapper script for opencode-memory MCP server
+# Connects to the HTTP server via mcp-remote
+#
+# Prerequisites:
+#   Linux: systemctl --user enable --now opencode-memory
+#   macOS: launchctl load ~/Library/LaunchAgents/com.opencode.memory.plist
+
+set -e
+
+SERVER_URL="http://localhost:9824"
+
+# Quick health check (fail fast if server not running)
+if ! curl -s --max-time 1 "$SERVER_URL/health" >/dev/null 2>&1; then
+    OS="$(uname -s)"
+    case "$OS" in
+        Linux*)
+            echo "Memory server not running. Start it with: systemctl --user start opencode-memory" >&2
+            ;;
+        Darwin*)
+            echo "Memory server not running. Start it with: launchctl load ~/Library/LaunchAgents/com.opencode.memory.plist" >&2
+            ;;
+        *)
+            echo "Memory server not running at $SERVER_URL" >&2
+            ;;
+    esac
+    exit 1
+fi
+
+# Resolve the mcp-remote command (locally installed is faster than npx)
+if [ -x "$HOME/.local/bin/mcp-remote" ]; then
+    MCP_REMOTE=("$HOME/.local/bin/mcp-remote")
+elif command -v mcp-remote >/dev/null 2>&1; then
+    MCP_REMOTE=(mcp-remote)
+else
+    MCP_REMOTE=(npx --yes mcp-remote)
+fi
+
+# Start mcp-remote with retry/backoff to absorb transient "Connection closed"
+# failures that occur when many subagents start concurrently (thundering herd
+# against the shared HTTP server). A clean long-running proxy never returns 0
+# here, so a fast non-zero exit means a startup-time connection drop worth retrying.
+MAX_ATTEMPTS=5
+attempt=1
+while true; do
+    start_ts=$(date +%s)
+    # Disable errexit around the proxy call; we handle its exit code explicitly.
+    set +e
+    "${MCP_REMOTE[@]}" "$SERVER_URL/mcp"
+    exit_code=$?
+    set -e
+    elapsed=$(( $(date +%s) - start_ts ))
+
+    # Exited cleanly, or ran long enough to be a real session (not a startup
+    # collision): propagate the exit code and stop.
+    if [ "$exit_code" -eq 0 ] || [ "$elapsed" -ge 10 ] || [ "$attempt" -ge "$MAX_ATTEMPTS" ]; then
+        exit "$exit_code"
+    fi
+
+    # Transient startup failure: back off (2s, 4s, 6s, 8s) and retry.
+    backoff=$(( attempt * 2 ))
+    echo "memory-mcp: mcp-remote exited ($exit_code) after ${elapsed}s on attempt ${attempt}; retrying in ${backoff}s" >&2
+    sleep "$backoff"
+    attempt=$(( attempt + 1 ))
+done

{opencode_semantic_memory-0.8.2 → opencode_semantic_memory-0.8.3}/src/opencode_memory/background_worker.py

@@ -1,5 +1,17 @@
 """Unified background worker for all memory processing.
 
+DEPRECATED: This module is superseded by the new job-based system in
+opencode_memory.jobs.worker. The new system provides:
+- Proper job queue with claim/timeout/retry
+- Circuit breaker protection for external APIs
+- Dead job tracking and alerting
+- Metrics collection for all job types
+
+Use instead: python -m opencode_memory.jobs.worker
+
+This file is kept for backwards compatibility and will be removed in v0.10.0.
+
+Legacy documentation:
 This runs as a separate process from the MCP server, handling:
 - Embedding computation and vector storage
 - Semantic deduplication
@@ -8,8 +20,6 @@ This runs as a separate process from the MCP server, handling:
 - Memory cleanup
 - LLM-based knowledge extraction
 - Concept extraction
-
-Run with: python -m opencode_memory.background_worker
 """
 
 import asyncio
@@ -979,6 +989,20 @@ Your response (one line only):"""
 
 def main():
     """Entry point for the background worker."""
+    import warnings
+
+    warnings.warn(
+        "background_worker is deprecated and will be removed in v0.10.0. "
+        "Use 'python -m opencode_memory.jobs.worker' or the 'opencode-memory-job-worker' "
+        "command instead. The new system provides job queuing, retries, and metrics.",
+        DeprecationWarning,
+        stacklevel=2,
+    )
+    logger.warning(
+        "DEPRECATION: background_worker is deprecated. "
+        "Use 'opencode-memory-job-worker' instead for the new job-based system."
+    )
+
     worker = BackgroundWorker()
 
     def handle_signal(signum, frame):

{opencode_semantic_memory-0.8.2 → opencode_semantic_memory-0.8.3}/src/opencode_memory/http_server.py

@@ -60,6 +60,10 @@ DEFAULT_PORT = int(os.environ.get("OPENCODE_MEMORY_PORT", "9824"))
 API_KEY = os.environ.get("OPENCODE_MEMORY_API_KEY")  # Optional auth
 RATE_LIMIT = int(os.environ.get("OPENCODE_MEMORY_RATE_LIMIT", "60"))  # per minute
 
+# Loopback addresses are exempt from rate limiting: all local opencode sessions
+# and their subagents connect over loopback and are trusted.
+_LOCAL_IPS = frozenset({"127.0.0.1", "::1", "localhost"})
+
 
 class RateLimiter:
     """Simple in-memory rate limiter using sliding window."""
@@ -69,20 +73,38 @@ class RateLimiter:
         self.window_seconds = 60
         self._requests: dict[str, list[float]] = defaultdict(list)
 
-    def _get_client_id(self, scope: dict) -> str:
-        """Get client identifier from request."""
-        # Use X-Forwarded-For if behind proxy, otherwise use client IP
+    def _get_client_ip(self, scope: dict) -> str:
+        """Get the originating IP for a request."""
         headers = dict(scope.get("headers", []))
         forwarded = headers.get(b"x-forwarded-for", b"").decode()
         if forwarded:
-            client_ip = forwarded.split(",")[0].strip()
-        else:
-            client = scope.get("client", ("unknown", 0))
-            client_ip = client[0] if client else "unknown"
-        return client_ip
+            return forwarded.split(",")[0].strip()
+        client = scope.get("client", ("unknown", 0))
+        return client[0] if client else "unknown"
+
+    def _get_client_id(self, scope: dict) -> str:
+        """Get client identifier for rate limiting.
+
+        Prefers the opencode session header so that multiple concurrent
+        sessions/subagents on the same host get independent rate-limit
+        buckets rather than all sharing a single localhost IP bucket
+        (which caused startup bursts to trip the limit and drop the
+        mcp-remote connection). Falls back to client IP.
+        """
+        headers = dict(scope.get("headers", []))
+        session_id = headers.get(b"x-opencode-session", b"").decode()
+        if session_id:
+            return f"session:{session_id[:16]}"
+        return self._get_client_ip(scope)
 
     def is_allowed(self, scope: dict) -> tuple[bool, int]:
         """Check if request is allowed. Returns (allowed, retry_after_seconds)."""
+        # Local traffic is trusted and must not be rate limited: all opencode
+        # sessions and subagents connect over loopback, and throttling them
+        # only serves to break legitimate concurrent usage.
+        if self._get_client_ip(scope) in _LOCAL_IPS:
+            return True, 0
+
         client_id = self._get_client_id(scope)
         now = time.time()
         window_start = now - self.window_seconds

opencode_semantic_memory-0.8.3/src/opencode_memory/jobs/__init__.py

@@ -0,0 +1,64 @@
+"""Background job queue system (Sidekiq-like)."""
+
+from opencode_memory.jobs.circuit_breaker import (
+    CircuitBreakerManager,
+    CircuitBreakerOpenError,
+)
+from opencode_memory.jobs.executor import (
+    JobExecutor,
+    JobExecutorBuilder,
+    ResourceLimits,
+)
+from opencode_memory.jobs.handlers import JobHandlers
+from opencode_memory.jobs.queue import (
+    DEFAULT_CONFIGS,
+    JobConfigRegistry,
+    JobQueue,
+)
+from opencode_memory.jobs.scheduler import (
+    DEFAULT_SCHEDULES,
+    JobScheduler,
+    ScheduleConfig,
+)
+from opencode_memory.jobs.worker import BackgroundJobWorker
+from opencode_memory.models import (
+    BackgroundJob,
+    CircuitBreaker,
+    CircuitBreakerState,
+    DeadJob,
+    JobConfig,
+    JobMetrics,
+    JobStatus,
+    ResourceType,
+)
+
+__all__ = [
+    # Queue
+    "JobQueue",
+    "JobConfigRegistry",
+    "DEFAULT_CONFIGS",
+    # Scheduler
+    "JobScheduler",
+    "ScheduleConfig",
+    "DEFAULT_SCHEDULES",
+    # Executor
+    "JobExecutor",
+    "JobExecutorBuilder",
+    "ResourceLimits",
+    # Handlers
+    "JobHandlers",
+    # Worker
+    "BackgroundJobWorker",
+    # Circuit Breaker
+    "CircuitBreakerManager",
+    "CircuitBreakerOpenError",
+    # Models
+    "BackgroundJob",
+    "CircuitBreaker",
+    "CircuitBreakerState",
+    "DeadJob",
+    "JobConfig",
+    "JobMetrics",
+    "JobStatus",
+    "ResourceType",
+]

opencode_semantic_memory-0.8.3/src/opencode_memory/jobs/circuit_breaker.py

@@ -0,0 +1,237 @@
+"""Circuit breaker pattern for protecting external resources."""
+
+import logging
+from collections.abc import Generator
+from contextlib import contextmanager
+from datetime import UTC, datetime
+
+from opencode_memory.models import (
+    CircuitBreaker as CircuitBreakerModel,
+)
+from opencode_memory.models import (
+    CircuitBreakerState,
+    Memory,
+    MemoryCategory,
+    ResourceType,
+)
+from opencode_memory.storage.sqlite import SQLiteStorage
+
+logger = logging.getLogger(__name__)
+
+
+class CircuitBreakerOpenError(Exception):
+    """Raised when the circuit breaker is open."""
+
+    def __init__(self, resource_type: str, message: str | None = None):
+        self.resource_type = resource_type
+        super().__init__(message or f"Circuit breaker for {resource_type} is OPEN")
+
+
+class CircuitBreakerManager:
+    """Manages circuit breakers for different resource types.
+
+    The circuit breaker pattern prevents cascade failures by:
+    1. CLOSED: Normal operation, requests allowed
+    2. OPEN: After N failures, block all requests for a timeout period
+    3. HALF_OPEN: After timeout, allow one test request
+
+    State persistence is backed by SQLite for durability across restarts.
+    When a circuit breaker opens, it creates a blocker in memory for boot context visibility.
+    """
+
+    # Configuration
+    FAILURE_THRESHOLD = 5  # Failures before opening
+    RESET_TIMEOUT_SECONDS = 300  # 5 minutes before trying half-open
+    HALF_OPEN_MAX_ATTEMPTS = 2  # Successes needed to close from half-open
+
+    def __init__(self, sqlite: SQLiteStorage):
+        self.sqlite = sqlite
+        self._breakers: dict[ResourceType, CircuitBreakerModel] = {}
+
+    def _get_breaker(self, resource_type: ResourceType) -> CircuitBreakerModel:
+        """Get or create a circuit breaker for a resource type."""
+        if resource_type not in self._breakers:
+            self._breakers[resource_type] = self.sqlite.circuit_breaker_get(resource_type.value)
+        return self._breakers[resource_type]
+
+    def _save_breaker(self, breaker: CircuitBreakerModel) -> None:
+        """Save breaker state to database."""
+        self.sqlite.circuit_breaker_update(breaker)
+        self._breakers[breaker.resource_type] = breaker
+
+    def can_execute(self, resource_type: ResourceType) -> bool:
+        """Check if requests are allowed for this resource.
+
+        Returns True if the circuit is closed or half-open.
+        """
+        breaker = self._get_breaker(resource_type)
+
+        if breaker.state == CircuitBreakerState.CLOSED:
+            return True
+
+        if breaker.state == CircuitBreakerState.OPEN:
+            # Check if timeout has elapsed
+            if breaker.opened_at:
+                elapsed = (datetime.now(UTC) - breaker.opened_at).total_seconds()
+                if elapsed >= self.RESET_TIMEOUT_SECONDS:
+                    # Transition to half-open
+                    breaker.state = CircuitBreakerState.HALF_OPEN
+                    breaker.half_open_attempts = 0
+                    self._save_breaker(breaker)
+                    logger.info(f"{resource_type.value} circuit breaker entering half-open state")
+                    return True
+            return False
+
+        # Half-open: allow requests for testing
+        return True
+
+    def record_success(self, resource_type: ResourceType) -> None:
+        """Record a successful call to the resource."""
+        breaker = self._get_breaker(resource_type)
+        breaker.last_success_at = datetime.now(UTC)
+
+        if breaker.state == CircuitBreakerState.HALF_OPEN:
+            breaker.half_open_attempts += 1
+            if breaker.half_open_attempts >= self.HALF_OPEN_MAX_ATTEMPTS:
+                # Close the circuit
+                breaker.state = CircuitBreakerState.CLOSED
+                breaker.failure_count = 0
+                breaker.opened_at = None
+                logger.info(f"{resource_type.value} circuit breaker CLOSED (recovered)")
+                self._resolve_blocker(resource_type)
+
+        elif breaker.state == CircuitBreakerState.CLOSED:
+            # Reset failure count on success
+            if breaker.failure_count > 0:
+                breaker.failure_count = 0
+
+        self._save_breaker(breaker)
+
+    def record_failure(self, resource_type: ResourceType, error: Exception) -> None:
+        """Record a failed call to the resource."""
+        breaker = self._get_breaker(resource_type)
+        now = datetime.now(UTC)
+
+        breaker.failure_count += 1
+        breaker.last_failure_at = now
+
+        if breaker.state == CircuitBreakerState.HALF_OPEN:
+            # Failure in half-open: go back to open
+            breaker.state = CircuitBreakerState.OPEN
+            breaker.opened_at = now
+            breaker.half_open_attempts = 0
+            logger.warning(f"{resource_type.value} circuit breaker re-OPENED from half-open: {error}")
+
+        elif breaker.state == CircuitBreakerState.CLOSED:
+            if breaker.failure_count >= self.FAILURE_THRESHOLD:
+                # Trip the breaker
+                breaker.state = CircuitBreakerState.OPEN
+                breaker.opened_at = now
+                logger.error(
+                    f"{resource_type.value} circuit breaker OPENED after {breaker.failure_count} failures: {error}"
+                )
+                self._create_blocker(resource_type, error)
+
+        self._save_breaker(breaker)
+
+    def _create_blocker(self, resource_type: ResourceType, error: Exception) -> None:
+        """Create a blocker memory when circuit opens."""
+        content = (
+            f"Circuit breaker for {resource_type.value.upper()} is OPEN.\n\n"
+            f"The {resource_type.value} service experienced {self.FAILURE_THRESHOLD} "
+            f"consecutive failures and has been temporarily disabled.\n\n"
+            f"Last error: {type(error).__name__}: {error}\n\n"
+            f"The circuit will automatically test recovery in "
+            f"{self.RESET_TIMEOUT_SECONDS // 60} minutes."
+        )
+
+        memory = Memory(
+            category=MemoryCategory.BLOCKER,
+            content=content,
+            what=f"{resource_type.value.upper()} circuit breaker OPEN",
+            why="Prevent cascade failures from repeated service errors",
+            entities=[f"circuit_breaker:{resource_type.value}"],
+        )
+        self.sqlite.insert_memory(memory)
+
+    def _resolve_blocker(self, resource_type: ResourceType) -> None:
+        """Resolve the blocker when circuit closes."""
+        # Find and resolve the blocker
+        with self.sqlite._get_conn() as conn:
+            cursor = conn.execute(
+                """
+                UPDATE memories
+                SET resolved_at = ?
+                WHERE category = 'blocker'
+                  AND content LIKE ?
+                  AND resolved_at IS NULL
+                """,
+                (
+                    datetime.now(UTC).isoformat(),
+                    f"%circuit breaker for {resource_type.value.upper()}%",
+                ),
+            )
+            if cursor.rowcount > 0:
+                logger.info(f"Resolved {resource_type.value} circuit breaker blocker")
+
+    def get_state(self, resource_type: ResourceType) -> CircuitBreakerState:
+        """Get the current state of a circuit breaker."""
+        breaker = self._get_breaker(resource_type)
+        return breaker.state
+
+    def get_all_states(self) -> dict[str, CircuitBreakerState]:
+        """Get all circuit breaker states."""
+        return self.sqlite.circuit_breaker_get_all()
+
+    def force_open(self, resource_type: ResourceType, reason: str) -> None:
+        """Manually open a circuit breaker."""
+        breaker = self._get_breaker(resource_type)
+        breaker.state = CircuitBreakerState.OPEN
+        breaker.opened_at = datetime.now(UTC)
+        self._save_breaker(breaker)
+
+        # Create a blocker
+        memory = Memory(
+            category=MemoryCategory.BLOCKER,
+            content=(f"Circuit breaker for {resource_type.value.upper()} manually OPENED.\n\nReason: {reason}"),
+            what=f"{resource_type.value.upper()} circuit breaker manually opened",
+            why=reason,
+            entities=[f"circuit_breaker:{resource_type.value}"],
+        )
+        self.sqlite.insert_memory(memory)
+        logger.warning(f"{resource_type.value} circuit breaker manually OPENED: {reason}")
+
+    def force_close(self, resource_type: ResourceType) -> None:
+        """Manually close a circuit breaker."""
+        breaker = self._get_breaker(resource_type)
+        breaker.state = CircuitBreakerState.CLOSED
+        breaker.failure_count = 0
+        breaker.opened_at = None
+        breaker.half_open_attempts = 0
+        self._save_breaker(breaker)
+        self._resolve_blocker(resource_type)
+        logger.info(f"{resource_type.value} circuit breaker manually CLOSED")
+
+    @contextmanager
+    def protect(self, resource_type: ResourceType) -> Generator[None, None, None]:
+        """Context manager for protected calls.
+
+        Usage:
+            with circuit_breaker.protect(ResourceType.LLM):
+                result = call_llm(...)
+
+        Raises:
+            CircuitBreakerOpenError: If the circuit is open
+        """
+        if not self.can_execute(resource_type):
+            raise CircuitBreakerOpenError(resource_type.value)
+
+        try:
+            yield
+            self.record_success(resource_type)
+        except CircuitBreakerOpenError:
+            # Re-raise without recording as failure
+            raise
+        except Exception as e:
+            self.record_failure(resource_type, e)
+            raise