surreal-memory 2.3.2__tar.gz → 2.5.0__tar.gz

{surreal_memory-2.3.2 → surreal_memory-2.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: surreal-memory
-Version: 2.3.2
+Version: 2.5.0
 Summary: Reflex-based memory system for AI agents with SurrealDB backend — retrieval through activation, not search
 Project-URL: Homepage, https://github.com/acidkill/surreal-memory
 Project-URL: Documentation, https://github.com/acidkill/surreal-memory/blob/main/docs/getting-started/quickstart.md
@@ -38,7 +38,7 @@ Requires-Dist: mem0ai>=0.1.0; extra == 'all'
 Requires-Dist: python-multipart>=0.0.22; extra == 'all'
 Requires-Dist: pyvi>=0.1; extra == 'all'
 Requires-Dist: spacy>=3.6; extra == 'all'
-Requires-Dist: surrealdb>=0.4.0; extra == 'all'
+Requires-Dist: surrealdb<3.0.0,>=2.0.0; extra == 'all'
 Requires-Dist: underthesea>=6.0; extra == 'all'
 Requires-Dist: uvicorn[standard]>=0.23; extra == 'all'
 Provides-Extra: chromadb
@@ -104,7 +104,7 @@ Requires-Dist: httpx>=0.24; extra == 'server'
 Requires-Dist: python-multipart>=0.0.22; extra == 'server'
 Requires-Dist: uvicorn[standard]>=0.23; extra == 'server'
 Provides-Extra: surrealdb
-Requires-Dist: surrealdb>=0.4.0; extra == 'surrealdb'
+Requires-Dist: surrealdb<3.0.0,>=2.0.0; extra == 'surrealdb'
 Provides-Extra: watch
 Requires-Dist: watchdog>=4.0.0; extra == 'watch'
 Description-Content-Type: text/markdown
@@ -217,7 +217,7 @@ smem recall "auth bug"
 
 ## 3 Tools. That's It.
 
-53 MCP tools are available, but you only need three:
+56 MCP tools are available, but you only need three:
 
 | Tool | What it does |
 |------|-------------|
@@ -233,7 +233,7 @@ Everything else — sessions, context loading, habit tracking, maintenance — w
 
 ```
                     ┌──────────────────────────────┐
-                    │       MCP Server (53 tools)   │
+                    │       MCP Server (56 tools)   │
                     └──────────┬───────────────────┘
                                │
                     ┌──────────▼───────────────────┐

{surreal_memory-2.3.2 → surreal_memory-2.5.0}/README.md

@@ -106,7 +106,7 @@ smem recall "auth bug"
 
 ## 3 Tools. That's It.
 
-53 MCP tools are available, but you only need three:
+56 MCP tools are available, but you only need three:
 
 | Tool | What it does |
 |------|-------------|
@@ -122,7 +122,7 @@ Everything else — sessions, context loading, habit tracking, maintenance — w
 
 ```
                     ┌──────────────────────────────┐
-                    │       MCP Server (53 tools)   │
+                    │       MCP Server (56 tools)   │
                     └──────────┬───────────────────┘
                                │
                     ┌──────────▼───────────────────┐

{surreal_memory-2.3.2 → surreal_memory-2.5.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "surreal-memory"
-version = "2.3.2"
+version = "2.5.0"
 description = "Reflex-based memory system for AI agents with SurrealDB backend — retrieval through activation, not search"
 readme = "README.md"
 license = "MIT"
@@ -52,7 +52,7 @@ server = [
     "httpx>=0.24",
 ]
 surrealdb = [
-    "surrealdb>=0.4.0",
+    "surrealdb>=2.0.0,<3.0.0",
 ]
 encryption = [
     "cryptography>=46.0.5",

{surreal_memory-2.3.2 → surreal_memory-2.5.0}/src/surreal_memory/__init__.py

@@ -16,7 +16,7 @@ from surreal_memory.engine.encoder import EncodingResult, MemoryEncoder
 from surreal_memory.engine.reflex_activation import CoActivation, ReflexActivation
 from surreal_memory.engine.retrieval import DepthLevel, ReflexPipeline, RetrievalResult
 
-__version__ = "2.3.2"
+__version__ = "2.5.0"
 
 __all__ = [
     "__version__",

{surreal_memory-2.3.2 → surreal_memory-2.5.0}/src/surreal_memory/cli/commands/config_cmd.py

@@ -13,7 +13,7 @@ config_app = typer.Typer(help="Configuration management")
 def preset_cmd(
     name: Annotated[
         str,
-        typer.Argument(help="Preset name: safe-cost, balanced, max-recall"),
+        typer.Argument(help="Preset name: safe-cost, balanced, max-recall, chat-heavy"),
     ] = "",
     list_available: Annotated[
         bool,

{surreal_memory-2.3.2 → surreal_memory-2.5.0}/src/surreal_memory/config_presets.py

@@ -1,6 +1,6 @@
 """Static configuration presets for Surreal-Memory.
 
-Three built-in profiles that configure brain behavior, maintenance,
+Four built-in profiles that configure brain behavior, maintenance,
 and retrieval for different use cases. Presets are static dicts
 (not a plugin system) to keep the surface simple and predictable.
 
@@ -82,16 +82,38 @@ MAX_RECALL: dict[str, dict[str, Any]] = {
     },
 }
 
+CHAT_HEAVY: dict[str, dict[str, Any]] = {
+    "brain": {
+        "decay_rate": 0.15,
+        "reinforcement_delta": 0.05,
+        "activation_threshold": 0.25,
+        "max_spread_hops": 3,
+        "max_context_tokens": 800,
+        "freshness_weight": 0.25,
+    },
+    "maintenance": {
+        "auto_consolidate": True,
+        "check_interval": 20,
+        "auto_consolidate_strategies": ["prune", "merge"],
+        "consolidate_cooldown_minutes": 20,
+    },
+    "eternal": {
+        "max_context_tokens": 64_000,
+    },
+}
+
 _PRESETS: dict[str, dict[str, dict[str, Any]]] = {
     "safe-cost": SAFE_COST,
     "balanced": BALANCED,
     "max-recall": MAX_RECALL,
+    "chat-heavy": CHAT_HEAVY,
 }
 
 _DESCRIPTIONS: dict[str, str] = {
     "safe-cost": "Lower token usage, faster decay, aggressive pruning",
     "balanced": "Default settings — good all-around performance",
     "max-recall": "Maximum retention, deeper retrieval, conservative pruning",
+    "chat-heavy": "Conversational agents (Telegram/Discord/Slack) — fast decay, recent-biased, compact",
 }
 
 

{surreal_memory-2.3.2 → surreal_memory-2.5.0}/src/surreal_memory/core/fiber.py

@@ -7,6 +7,7 @@ from datetime import datetime
 from typing import Any
 from uuid import uuid4
 
+from surreal_memory.utils.tag_normalizer import normalize_tags_lower
 from surreal_memory.utils.timeutils import utcnow
 
 
@@ -125,6 +126,10 @@ class Fiber:
         if tags is not None and not auto_tags and not agent_tags:
             effective_agent = tags
 
+        # Normalize case at ingestion boundary so storage is always lowercase
+        effective_auto = normalize_tags_lower(effective_auto)
+        effective_agent = normalize_tags_lower(effective_agent)
+
         return cls(
             id=fiber_id or str(uuid4()),
             neuron_ids=neuron_ids,

{surreal_memory-2.3.2 → surreal_memory-2.5.0}/src/surreal_memory/core/synapse.py

@@ -94,6 +94,8 @@ class Direction(StrEnum):
 
     UNIDIRECTIONAL = "uni"  # One-way: source -> target
     BIDIRECTIONAL = "bi"  # Two-way: source <-> target
+    FORWARD = "forward"  # legacy rows; also _row_to_synapse default in surrealdb store
+    BACKWARD = "backward"  # legacy rows: target -> source
 
 
 # Synapse types that are typically bidirectional

{surreal_memory-2.3.2 → surreal_memory-2.5.0}/src/surreal_memory/engine/brain_transplant.py

@@ -79,9 +79,14 @@ def _fiber_matches_tags(
     fiber: dict[str, Any],
     required_tags: frozenset[str],
 ) -> bool:
-    """Return True if the fiber carries ANY of the required tags."""
-    fiber_tags = set(fiber.get("tags", []))
-    return bool(fiber_tags & required_tags)
+    """Return True if the fiber carries ANY of the required tags.
+
+    Both the fiber's tags and the query tags are lowercased before comparison
+    so that "KB" matches a fiber tagged "kb" regardless of original casing.
+    """
+    fiber_tags = {t.lower() for t in fiber.get("tags", [])}
+    normalized_required = frozenset(t.lower() for t in required_tags)
+    return bool(fiber_tags & normalized_required)
 
 
 def _fiber_matches_salience(

{surreal_memory-2.3.2 → surreal_memory-2.5.0}/src/surreal_memory/engine/consolidation.py

@@ -598,6 +598,14 @@ class ConsolidationEngine:
             states = await self._storage.get_neuron_states_batch(batch_ids)
 
             for neuron in batch:
+                # Never auto-prune pinned neurons, whether isolated (orphan) or
+                # dead. The dead-neuron path already honored pinned, but the
+                # orphan path short-circuited above it, so pinned isolated
+                # neurons were permanently deleted. Hoist the guard so it
+                # protects both paths.
+                if neuron.id in pinned_neuron_ids:
+                    continue
+
                 is_orphan = (
                     neuron.id not in connected_neuron_ids and neuron.id not in fiber_neuron_ids
                 )
@@ -606,9 +614,7 @@ class ConsolidationEngine:
                     orphan_ids.append(neuron.id)
                     continue
 
-                # Dead neuron: has connections but never accessed, old enough, not pinned
-                if neuron.id in pinned_neuron_ids:
-                    continue
+                # Dead neuron: has connections but never accessed, old enough
                 state = states.get(neuron.id)
                 freq = state.access_frequency if state else 0
                 if freq > 0:

{surreal_memory-2.3.2 → surreal_memory-2.5.0}/src/surreal_memory/engine/embedding/gemini_embedding.py

@@ -61,7 +61,21 @@ class GeminiEmbedding(EmbeddingProvider):
                     "Install it with: pip install 'surreal-memory[embeddings-gemini]'"
                 ) from exc
 
-            self._client = genai.Client(api_key=self._api_key)
+            # Optional gateway routing: honour GOOGLE_GEMINI_BASE_URL (the
+            # SDK env var) but build the client explicitly, because our
+            # gateway path already pins the API version — api_version must be
+            # emptied or the SDK doubles it ('/v1beta/v1beta/...').
+            base_url = os.getenv("GOOGLE_GEMINI_BASE_URL")
+            if base_url:
+                self._client = genai.Client(
+                    api_key=self._api_key,
+                    http_options={
+                        "base_url": base_url,
+                        "api_version": os.getenv("GOOGLE_GEMINI_API_VERSION", ""),
+                    },
+                )
+            else:
+                self._client = genai.Client(api_key=self._api_key)
 
         return self._client
 
@@ -94,13 +108,19 @@ class GeminiEmbedding(EmbeddingProvider):
         batch_size = 100
         for i in range(0, len(texts), batch_size):
             chunk = texts[i : i + batch_size]
-            # functools.partial binds `chunk` at creation (satisfies ruff B023,
-            # no loop-var late binding) and types cleanly for mypy.
+            # google-genai >= 2.0 folds a plain list[str] into ONE multi-part
+            # content (N texts -> 1 embedding); give each text its own content
+            # so the API returns one embedding per text. Plain ContentDicts are
+            # used instead of types.Content to avoid importing google.genai here
+            # (keeps this path working when the SDK is mocked/absent in tests).
+            chunk_contents = [{"parts": [{"text": t}]} for t in chunk]
+            # functools.partial binds the chunk at creation (satisfies ruff
+            # B023, no loop-var late binding) and types cleanly for mypy.
             response = await call_with_retry(
                 partial(
                     client.aio.models.embed_content,
                     model=self._model,
-                    contents=chunk,
+                    contents=chunk_contents,
                     config={"task_type": self._task_type},
                 ),
                 provider="gemini",

{surreal_memory-2.3.2 → surreal_memory-2.5.0}/src/surreal_memory/engine/encoder.py

@@ -64,6 +64,9 @@ class EncodingResult:
         neurons_created: List of newly created neurons
         neurons_linked: List of existing neuron IDs that were linked
         synapses_created: List of newly created synapses
+        extraction_stats: Optional concept-extraction counters when callers
+            opt in via verbose_extraction. Surface schema:
+            ``{"dropped_short", "dropped_noise", "dropped_duplicate_entity"}``.
     """
 
     fiber: Fiber
@@ -71,6 +74,7 @@ class EncodingResult:
     neurons_linked: list[str]
     synapses_created: list[Synapse]
     conflicts_detected: int = 0
+    extraction_stats: dict[str, int] | None = None
 
 
 def build_default_pipeline(
@@ -432,6 +436,11 @@ class MemoryEncoder:
             neurons_linked=ctx.neurons_linked,
             synapses_created=ctx.synapses_created,
             conflicts_detected=ctx.conflicts_detected,
+            extraction_stats={
+                "dropped_short": ctx.dropped_short,
+                "dropped_noise": ctx.dropped_noise,
+                "dropped_duplicate_entity": ctx.dropped_duplicate_entity,
+            },
         )
 
     async def _post_encode_neuro(self, anchor: Neuron) -> None:

{surreal_memory-2.3.2 → surreal_memory-2.5.0}/src/surreal_memory/engine/pipeline.py

@@ -72,6 +72,12 @@ class PipelineContext:
     # Entities stored here are first-mentions — not yet promoted to neurons
     deferred_entity_refs: list[str] = field(default_factory=list)
 
+    # Concept extraction observability — incremented by ExtractConceptNeuronsStep.
+    # Surfaced through EncodingResult.extraction_stats only when callers opt in.
+    dropped_short: int = 0
+    dropped_noise: int = 0
+    dropped_duplicate_entity: int = 0
+
 
 @runtime_checkable
 class PipelineStep(Protocol):

{surreal_memory-2.3.2 → surreal_memory-2.5.0}/src/surreal_memory/engine/pipeline_steps.py

@@ -320,11 +320,14 @@ class ExtractConceptNeuronsStep:
             kw_lower = kw.lower()
             # Minimum 4 chars — 3-char words produce too many noise concepts ("ai", "os")
             if len(kw_lower) < 4:
+                ctx.dropped_short += 1
                 return False
             if kw_lower in _NOISE_CONCEPTS:
+                ctx.dropped_noise += 1
                 return False
             # Skip if already captured as an entity neuron
             if kw_lower in entity_content:
+                ctx.dropped_duplicate_entity += 1
                 return False
             return True
 

{surreal_memory-2.3.2 → surreal_memory-2.5.0}/src/surreal_memory/hooks/post_tool_use.py

@@ -7,8 +7,15 @@ Usage as Claude Code hook:
     Receives JSON on stdin with tool_name, tool_input, tool_output fields.
     Writes one JSONL line to ~/.surrealmemory/tool_events.jsonl.
 
-This hook does NOT access SQLite or perform encoding — all processing
+This hook does NOT access SurrealDB or perform encoding — all processing
 is deferred to the consolidation cycle.
+
+Performance design (ac2a001 perf subset):
+- Zero heavy imports on the hot path (stdlib only except optional tomllib)
+- _NOISE_TOOLS fast-path skips the highest-frequency no-value tools
+- Lock-safe JSONL append via fcntl.flock (POSIX) with a write-then-rename
+  fallback for platforms without flock
+- Session ID: checks CLAUDE_SESSION_ID and CODEX_SESSION_ID
 """
 
 from __future__ import annotations
@@ -18,6 +25,7 @@ import logging
 import os
 import sys
 import time
+from datetime import UTC, datetime
 from pathlib import Path
 from typing import Any
 
@@ -28,6 +36,29 @@ _MAX_ARGS_CHARS = 200
 # Max size for stdout response JSON
 _MAX_TOOL_OUTPUT_PREVIEW = 100
 
+# High-frequency tools that never produce useful memory signal.
+# Checked before any config I/O for a fast zero-cost exit.
+_NOISE_TOOLS: frozenset[str] = frozenset(
+    {
+        "TodoRead",
+        "TodoWrite",
+        "WebSearch",
+        "WebFetch",
+        "mcp__Claude_Preview__preview_logs",
+        "mcp__Claude_Preview__preview_console_logs",
+        "mcp__Claude_Preview__preview_network",
+        "smem_recall",
+        "smem_session",
+        "smem_stats",
+        "smem_index",
+    }
+)
+
+
+def _get_session_id() -> str:
+    """Return the current session ID from env (Claude or Codex)."""
+    return os.environ.get("CLAUDE_SESSION_ID") or os.environ.get("CODEX_SESSION_ID", "")
+
 
 def _read_stdin() -> dict[str, Any]:
     """Read Claude Code PostToolUse hook JSON from stdin."""
@@ -41,49 +72,50 @@ def _read_stdin() -> dict[str, Any]:
         return {}
 
 
+def _get_data_dir() -> Path:
+    """Return the surreal-memory data directory."""
+    custom = os.environ.get("SURREAL_MEMORY_DIR", "")
+    return Path(custom) if custom else (Path.home() / ".surrealmemory")
+
+
 def _get_buffer_path() -> Path:
     """Get the JSONL buffer file path."""
-    data_dir = Path(os.environ.get("SURREAL_MEMORY_DIR", "")) or (Path.home() / ".surrealmemory")
-    return data_dir / "tool_events.jsonl"
+    return _get_data_dir() / "tool_events.jsonl"
 
 
-def _is_enabled() -> bool:
-    """Quick check if tool memory is enabled via config.
+def _read_tool_memory_config() -> dict[str, Any]:
+    """Read [tool_memory] section from config.toml once.
 
-    Reads only the [tool_memory] section from config.toml.
-    Defaults to True if config is missing or section absent.
+    Returns an empty dict if the config is missing, unreadable, or
+    the section is absent.  Called at most once per hook invocation.
     """
-    data_dir = Path(os.environ.get("SURREAL_MEMORY_DIR", "")) or (Path.home() / ".surrealmemory")
-    config_path = data_dir / "config.toml"
+    config_path = _get_data_dir() / "config.toml"
     if not config_path.exists():
-        return True
+        return {}
     try:
         import tomllib
 
         with open(config_path, "rb") as f:
             data = tomllib.load(f)
-        return bool(data.get("tool_memory", {}).get("enabled", True))
+        result: dict[str, Any] = data.get("tool_memory", {})
+        return result
     except Exception:
-        logger.debug("Failed to read tool_memory.enabled from config", exc_info=True)
-        return True
+        logger.debug("Failed to read tool_memory config", exc_info=True)
+        return {}
 
 
-def _get_blacklist() -> list[str]:
-    """Read blacklist from config.toml."""
-    data_dir = Path(os.environ.get("SURREAL_MEMORY_DIR", "")) or (Path.home() / ".surrealmemory")
-    config_path = data_dir / "config.toml"
-    if not config_path.exists():
-        return []
-    try:
-        import tomllib
+def _is_enabled(tm_cfg: dict[str, Any]) -> bool:
+    """Check if tool memory is enabled.
 
-        with open(config_path, "rb") as f:
-            data = tomllib.load(f)
-        bl = data.get("tool_memory", {}).get("blacklist", [])
-        return list(bl) if isinstance(bl, (list, tuple)) else []
-    except Exception:
-        logger.debug("Failed to read tool_memory.blacklist from config", exc_info=True)
-        return []
+    Defaults to True if the key is absent.
+    """
+    return bool(tm_cfg.get("enabled", True))
+
+
+def _get_blacklist(tm_cfg: dict[str, Any]) -> list[str]:
+    """Return the blacklist from [tool_memory] config."""
+    bl = tm_cfg.get("blacklist", [])
+    return list(bl) if isinstance(bl, (list, tuple)) else []
 
 
 def _truncate_args(tool_input: Any) -> str:
@@ -97,16 +129,18 @@ def _truncate_args(tool_input: Any) -> str:
     return raw[:_MAX_ARGS_CHARS]
 
 
-def _format_event(hook_input: dict[str, Any]) -> dict[str, Any]:
-    """Format hook input into a JSONL event dict."""
-    from surreal_memory.utils.timeutils import utcnow
+def _utcnow_iso() -> str:
+    """Return current UTC time as naive ISO string (stdlib only, no imports)."""
+    return datetime.now(UTC).replace(tzinfo=None).isoformat()
+
 
+def _format_event(hook_input: dict[str, Any]) -> dict[str, Any]:
+    """Format hook input into a JSONL event dict (stdlib only)."""
     tool_name = hook_input.get("tool_name", hook_input.get("tool", ""))
     server_name = hook_input.get("server_name", "")
     tool_input = hook_input.get("tool_input", {})
     tool_error = hook_input.get("tool_error")
     duration_ms = hook_input.get("duration_ms", 0)
-    session_id = os.environ.get("CLAUDE_SESSION_ID", "")
 
     return {
         "tool_name": str(tool_name),
@@ -114,22 +148,35 @@ def _format_event(hook_input: dict[str, Any]) -> dict[str, Any]:
         "args_summary": _truncate_args(tool_input),
         "success": tool_error is None,
         "duration_ms": int(duration_ms) if isinstance(duration_ms, (int, float)) else 0,
-        "session_id": session_id,
+        "session_id": _get_session_id(),
         "task_context": "",  # Populated by processing engine if session is active
-        "created_at": utcnow().isoformat(),
+        "created_at": _utcnow_iso(),
     }
 
 
 def _append_to_buffer(event: dict[str, Any], buffer_path: Path) -> bool:
-    """Append one JSONL line to the buffer file.
+    """Append one JSONL line to the buffer file, lock-safe.
 
+    Uses fcntl.flock (POSIX) when available.  Falls back to a plain
+    append on platforms that lack flock (Windows, some embedded).
     Returns True on success, False on failure.
     """
     try:
         buffer_path.parent.mkdir(parents=True, exist_ok=True)
-        line = json.dumps(event, ensure_ascii=False, default=str)
-        with open(buffer_path, "a", encoding="utf-8") as f:
-            f.write(line + "\n")
+        line = json.dumps(event, ensure_ascii=False, default=str) + "\n"
+        try:
+            import fcntl
+
+            with open(buffer_path, "a", encoding="utf-8") as f:
+                fcntl.flock(f, fcntl.LOCK_EX)
+                try:
+                    f.write(line)
+                finally:
+                    fcntl.flock(f, fcntl.LOCK_UN)
+        except ImportError:
+            # Non-POSIX platform — plain append (best-effort)
+            with open(buffer_path, "a", encoding="utf-8") as f:
+                f.write(line)
         return True
     except OSError:
         return False
@@ -155,26 +202,31 @@ def main() -> None:
     """Entry point for the PostToolUse hook."""
     start = time.monotonic()
 
-    # Fast exit if disabled
-    if not _is_enabled():
-        # Output empty JSON for hook response
-        sys.stdout.write("{}\n")
-        return
-
     hook_input = _read_stdin()
     if not hook_input:
         sys.stdout.write("{}\n")
         return
 
-    tool_name = hook_input.get("tool_name", hook_input.get("tool", ""))
+    tool_name = str(hook_input.get("tool_name", hook_input.get("tool", "")))
     if not tool_name:
         sys.stdout.write("{}\n")
         return
 
-    # Check blacklist
-    blacklist = _get_blacklist()
+    # Fast-path: skip high-frequency noise tools before any config I/O
+    if tool_name in _NOISE_TOOLS:
+        sys.stdout.write("{}\n")
+        return
+
+    # Read config once; derive enabled + blacklist from it
+    tm_cfg = _read_tool_memory_config()
+
+    if not _is_enabled(tm_cfg):
+        sys.stdout.write("{}\n")
+        return
+
+    blacklist = _get_blacklist(tm_cfg)
     for prefix in blacklist:
-        if str(tool_name).startswith(prefix):
+        if tool_name.startswith(prefix):
             sys.stdout.write("{}\n")
             return
 
@@ -183,7 +235,7 @@ def main() -> None:
     buffer_path = _get_buffer_path()
     _append_to_buffer(event, buffer_path)
 
-    # Periodic buffer rotation check (every ~100 calls, cheap stat check)
+    # Periodic buffer rotation check (cheap stat check)
     try:
         if buffer_path.exists() and buffer_path.stat().st_size > 5_000_000:  # > 5MB
             _check_buffer_rotation(buffer_path)