memkernel 0.2.0__py3-none-any.whl

memkernel/mcp/tools/memory.py

@@ -0,0 +1,148 @@
+# Copyright (c) 2026 EGOGE
+# SPDX-License-Identifier: Apache-2.0
+"""MCP tool registrations: retain, recall, get, forget, promote, consolidate, forget_expired."""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from mcp.server.fastmcp import FastMCP  # noqa: TC002 — runtime import required by FastMCP introspection
+from mcp.server.fastmcp.server import Context  # noqa: TC002
+
+from memkernel.core.models import MemoryTier, MemoryType
+
+
+def _service(ctx: Context[Any, Any, Any] | None) -> Any:
+    if ctx is None:
+        raise RuntimeError("Tool called outside of MCP request context")
+    return ctx.request_context.lifespan_context.memory_service
+
+
+def register_memory_tools(server: FastMCP[Any]) -> None:
+    """Register the memory tools on the server."""
+
+    @server.tool()
+    async def retain(
+        content: str,
+        type: str,  # noqa: A002
+        session_id: str | None = None,
+        checkpoint: bool = False,
+        source: str = "",
+        importance: float | None = None,
+        tags: list[str] | None = None,
+        supersedes: str | None = None,
+        ctx: Context[Any, Any, Any] | None = None,
+    ) -> str:
+        """Store a memory. Returns ref_id. Idempotent for identical content+type.
+
+        type: decision | convention | fact | code | procedure (long-term),
+              episode | observation (short-term), task (working memory).
+        importance: 0..1, defaults per type. supersedes: ref_id this memory replaces.
+        """
+        svc = _service(ctx)
+        ref_id = svc.retain(
+            content=content,
+            type=MemoryType(type),
+            session_id=session_id,
+            checkpoint=checkpoint,
+            source=source,
+            importance=importance,
+            tags=tags,
+            supersedes=supersedes,
+        )
+        return json.dumps({"ref_id": ref_id})
+
+    @server.tool()
+    async def recall(
+        query: str,
+        type: str | None = None,  # noqa: A002
+        tier: str | None = None,
+        session_id: str | None = None,
+        checkpoint_only: bool = False,
+        top_k: int | None = None,
+        ctx: Context[Any, Any, Any] | None = None,
+    ) -> str:
+        """Search memories by query. Returns ref_id + preview pairs ranked by relevance,
+        recency and importance. Use get(ref_id) for full content.
+
+        tier: working | short_term | long_term to restrict the search to one tier.
+        """
+        svc = _service(ctx)
+        results = svc.recall(
+            query=query,
+            type=MemoryType(type) if type else None,
+            tier=MemoryTier(tier) if tier else None,
+            session_id=session_id,
+            checkpoint_only=checkpoint_only,
+            top_k=top_k,
+        )
+        return json.dumps([ref.model_dump(mode="json") for ref in results])
+
+    @server.tool()
+    async def get(
+        ref_id: str,
+        ctx: Context[Any, Any, Any] | None = None,
+    ) -> str:
+        """Retrieve full content of a memory by ref_id."""
+        svc = _service(ctx)
+        memory = svc.get(ref_id)
+        if memory is None:
+            return json.dumps({"error": "not_found", "ref_id": ref_id})
+        return json.dumps(memory.model_dump(mode="json", exclude={"chunk_refs"}))
+
+    @server.tool()
+    async def forget(
+        ref_id: str,
+        ctx: Context[Any, Any, Any] | None = None,
+    ) -> str:
+        """Delete a memory by ref_id. Removes from all indexes."""
+        svc = _service(ctx)
+        deleted = svc.forget(ref_id)
+        return json.dumps({"ref_id": ref_id, "deleted": deleted})
+
+    @server.tool()
+    async def promote(
+        ref_id: str,
+        type: str,  # noqa: A002
+        content: str | None = None,
+        importance: float | None = None,
+        tags: list[str] | None = None,
+        ctx: Context[Any, Any, Any] | None = None,
+    ) -> str:
+        """Promote a short-term/working memory into a long-term type (decision, convention,
+        fact, code, procedure). Pass the distilled `content`; defaults to the original text.
+        Returns the new ref_id; the source memory is kept and linked via promoted_from.
+        """
+        svc = _service(ctx)
+        new_ref = svc.promote(
+            ref_id=ref_id,
+            type=MemoryType(type),
+            content=content,
+            importance=importance,
+            tags=tags,
+        )
+        if new_ref is None:
+            return json.dumps({"error": "not_found", "ref_id": ref_id})
+        return json.dumps({"ref_id": new_ref, "promoted_from": ref_id})
+
+    @server.tool()
+    async def consolidate(
+        session_id: str | None = None,
+        limit: int = 20,
+        ctx: Context[Any, Any, Any] | None = None,
+    ) -> str:
+        """Run a consolidation pass: purge expired short-term memories and return the
+        short-term/working memories most worth promoting (ranked by importance and how
+        often they were recalled). Call promote(ref_id, type, content) for the keepers.
+        """
+        svc = _service(ctx)
+        return json.dumps(svc.consolidate(session_id=session_id, limit=limit))
+
+    @server.tool()
+    async def forget_expired(
+        ctx: Context[Any, Any, Any] | None = None,
+    ) -> str:
+        """Delete every short-term/working memory whose retention window has lapsed."""
+        svc = _service(ctx)
+        return json.dumps({"deleted": svc.forget_expired()})

memkernel/mcp/tools/wiki.py

@@ -0,0 +1,59 @@
+# Copyright (c) 2026 EGOGE
+# SPDX-License-Identifier: Apache-2.0
+"""MCP tool registrations for the LLM-wiki bridge: wiki_export, wiki_import."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+from mcp.server.fastmcp import FastMCP  # noqa: TC002 — runtime import required by FastMCP introspection
+from mcp.server.fastmcp.server import Context  # noqa: TC002
+
+from memkernel.wiki.sync import WikiExporter, WikiImporter
+
+
+def _context(ctx: Context[Any, Any, Any] | None) -> Any:
+    if ctx is None:
+        raise RuntimeError("Tool called outside of MCP request context")
+    return ctx.request_context.lifespan_context
+
+
+def _resolve_wiki_dir(wiki_dir: str | None, settings: Any) -> Path:
+    resolved = wiki_dir or settings.wiki_dir
+    if not resolved:
+        raise ValueError("No wiki directory: pass wiki_dir or set MEMKERNEL_WIKI_DIR")
+    return Path(resolved)
+
+
+def register_wiki_tools(server: FastMCP[Any]) -> None:
+    """Register wiki_export and wiki_import on the server."""
+
+    @server.tool()
+    async def wiki_export(
+        wiki_dir: str | None = None,
+        ctx: Context[Any, Any, Any] | None = None,
+    ) -> str:
+        """Compile long-term memories (decision, convention, fact, procedure) that have no wiki
+        page yet into draft markdown pages under wiki_dir (default MEMKERNEL_WIKI_DIR), and
+        append them to index.md and log.md. Review and curate the drafts before marking stable.
+        """
+        app_ctx = _context(ctx)
+        exporter = WikiExporter(app_ctx.memory_service, _resolve_wiki_dir(wiki_dir, app_ctx.settings))
+        report = exporter.export()
+        return json.dumps({"written": report.written, "skipped": report.skipped})
+
+    @server.tool()
+    async def wiki_import(
+        wiki_dir: str | None = None,
+        ctx: Context[Any, Any, Any] | None = None,
+    ) -> str:
+        """Index every curated wiki page under wiki_dir (default MEMKERNEL_WIKI_DIR) as a
+        long-term memory so recall() can find it. Idempotent: unchanged pages dedup, changed
+        pages supersede their previous version.
+        """
+        app_ctx = _context(ctx)
+        importer = WikiImporter(app_ctx.memory_service, _resolve_wiki_dir(wiki_dir, app_ctx.settings))
+        report = importer.import_pages()
+        return json.dumps({"imported": report.imported, "skipped": report.skipped})

memkernel/memory/__init__.py

@@ -0,0 +1,2 @@
+# Copyright (c) 2026 EGOGE
+# SPDX-License-Identifier: Apache-2.0

memkernel/memory/dedup.py

@@ -0,0 +1,12 @@
+# Copyright (c) 2026 EGOGE
+# SPDX-License-Identifier: Apache-2.0
+"""Content-hash deduplication helpers."""
+
+from __future__ import annotations
+
+import hashlib
+
+
+def compute_content_hash(content: str) -> str:
+    """Return the SHA-256 hex digest of the UTF-8 encoded content string."""
+    return hashlib.sha256(content.encode("utf-8")).hexdigest()

memkernel/memory/lifecycle.py

@@ -0,0 +1,92 @@
+# Copyright (c) 2026 EGOGE
+# SPDX-License-Identifier: Apache-2.0
+"""Memory lifecycle helpers — decay, retention and expiry.
+
+Forgetting follows an Ebbinghaus-style model (as used by MemoryBank, Zhong et
+al. 2023): a memory's retention window grows with every recall, so knowledge
+that keeps being useful survives while untouched episodic noise fades.
+
+    effective_ttl = base_ttl * (1 + access_boost * access_count)
+
+Long-term memories never expire; they are curated by consolidation instead.
+"""
+
+from __future__ import annotations
+
+import math
+from datetime import UTC, datetime
+from typing import TYPE_CHECKING
+
+from memkernel.memory.types import get_half_life_days, get_ttl_days
+
+if TYPE_CHECKING:
+    from memkernel.config import MemKernelSettings
+    from memkernel.core.models import Memory, MemoryType
+
+
+def effective_ttl_days(
+    memory_type: MemoryType,
+    access_count: int,
+    settings: MemKernelSettings,
+) -> float | None:
+    """Return the retention window in days after access-based reinforcement, or None for no expiry."""
+    base = get_ttl_days(memory_type, settings)
+    if base is None:
+        return None
+    return base * (1.0 + settings.access_boost * max(0, access_count))
+
+
+def is_expired(
+    memory_type: MemoryType,
+    timestamp: datetime,
+    settings: MemKernelSettings,
+    access_count: int = 0,
+    last_accessed: datetime | None = None,
+) -> bool:
+    """Return True if the memory has exceeded its (reinforced) retention window.
+
+    The clock starts at the later of ``timestamp`` and ``last_accessed`` so a
+    recall refreshes the memory the way rehearsal does.
+    """
+    ttl_days = effective_ttl_days(memory_type, access_count, settings)
+    if ttl_days is None:
+        return False
+    anchor = timestamp
+    if last_accessed is not None and last_accessed > anchor:
+        anchor = last_accessed
+    age_days = (datetime.now(UTC) - anchor).total_seconds() / 86400.0
+    return age_days > ttl_days
+
+
+def memory_is_expired(memory: Memory, settings: MemKernelSettings) -> bool:
+    """Convenience wrapper over :func:`is_expired` for a full Memory record."""
+    return is_expired(
+        memory.type,
+        memory.timestamp,
+        settings,
+        access_count=memory.access_count,
+        last_accessed=memory.last_accessed,
+    )
+
+
+def recency_score(
+    memory_type: MemoryType,
+    timestamp: datetime,
+    settings: MemKernelSettings,
+    last_accessed: datetime | None = None,
+    now: datetime | None = None,
+) -> float:
+    """Return a recency factor in (0, 1]: exponential decay with a per-tier half-life.
+
+    Long-term memories have no recency decay (always 1.0) — a convention that
+    was set two years ago is as relevant as one set yesterday.
+    """
+    half_life = get_half_life_days(memory_type, settings)
+    if half_life is None or half_life <= 0:
+        return 1.0
+    anchor = timestamp
+    if last_accessed is not None and last_accessed > anchor:
+        anchor = last_accessed
+    current = now or datetime.now(UTC)
+    age_days = max(0.0, (current - anchor).total_seconds() / 86400.0)
+    return math.exp(-math.log(2.0) * age_days / half_life)

memkernel/memory/types.py

@@ -0,0 +1,40 @@
+# Copyright (c) 2026 EGOGE
+# SPDX-License-Identifier: Apache-2.0
+"""TTL and decay policies for each MemoryType / MemoryTier."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from memkernel.core.models import MemoryTier, MemoryType, tier_of
+
+if TYPE_CHECKING:
+    from memkernel.config import MemKernelSettings
+
+
+def get_ttl_days(memory_type: MemoryType, settings: MemKernelSettings) -> float | None:
+    """Return the base TTL in days for a given MemoryType, or None for no expiry.
+
+    Long-term types never expire. Short-term and working types read their
+    base TTL from settings so it is configurable at runtime.
+    """
+    tier = tier_of(memory_type)
+    if tier == MemoryTier.LONG_TERM:
+        return None
+    if memory_type == MemoryType.EPISODE:
+        return float(settings.episode_ttl_days)
+    if memory_type == MemoryType.OBSERVATION:
+        return float(settings.observation_ttl_days)
+    if memory_type == MemoryType.TASK:
+        return settings.task_ttl_hours / 24.0
+    return None  # pragma: no cover — every non-long-term type is handled above
+
+
+def get_half_life_days(memory_type: MemoryType, settings: MemKernelSettings) -> float | None:
+    """Return the recency half-life (days) used for recall scoring, or None for no recency decay."""
+    tier = tier_of(memory_type)
+    if tier == MemoryTier.WORKING:
+        return settings.task_ttl_hours / 24.0 / 2.0
+    if tier == MemoryTier.SHORT_TERM:
+        return float(settings.recency_half_life_days)
+    return None

memkernel/retrieval/__init__.py

@@ -0,0 +1,2 @@
+# Copyright (c) 2026 EGOGE
+# SPDX-License-Identifier: Apache-2.0

memkernel/retrieval/chunking.py

@@ -0,0 +1,244 @@
+# Copyright (c) 2026 EGOGE
+# SPDX-License-Identifier: Apache-2.0
+"""AST-aware code chunker implementing ChunkerPort.
+
+Splits code files at syntactic boundaries (functions, classes, methods) using
+tree-sitter.  Falls back to a sliding-window token chunker for unsupported
+languages or when tree-sitter is unavailable.
+"""
+
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from memkernel.config import MemKernelSettings
+
+logger = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# Language configuration
+# ---------------------------------------------------------------------------
+
+# Maps normalised language name -> tree-sitter language module import path.
+# MVP: Python only.  Add tree-sitter-javascript / tree-sitter-typescript deps
+# and extend this dict when JS/TS support is needed.
+_LANG_MODULE: dict[str, str] = {
+    "python": "tree_sitter_python",
+}
+
+# Boundary node types per language — AST node kinds at whose boundaries we
+# split.  Nodes within chunk_size are emitted as chunks; oversized nodes are
+# recursed into to find inner boundaries.
+_BOUNDARY_NODES: dict[str, frozenset[str]] = {
+    "python": frozenset({"function_definition", "class_definition", "decorated_definition"}),
+}
+
+# Maps file extensions to normalised language names.
+# Only extensions with a corresponding entry in _LANG_MODULE produce AST chunks.
+_EXT_TO_LANG: dict[str, str] = {
+    ".py": "python",
+}
+
+
+# ---------------------------------------------------------------------------
+# Token-based fallback (pure function, no tree-sitter dependency)
+# ---------------------------------------------------------------------------
+
+
+def _inner_boundary_nodes(node: Any, boundary_types: frozenset[str]) -> list[Any]:
+    """Return all boundary-type descendants of *node*, stopping at each one found.
+
+    The search is depth-first but does not descend into a node once that node
+    itself is a boundary — its own inner structure is handled by the recursive
+    _walk call that will be made for it later.
+    """
+    result: list[Any] = []
+    for child in node.children:
+        if child.type in boundary_types:
+            result.append(child)
+        else:
+            result.extend(_inner_boundary_nodes(child, boundary_types))
+    return result
+
+
+def _token_chunks(text: str, chunk_size: int, chunk_overlap: int) -> list[str]:
+    """Sliding-window character chunker.
+
+    Returns ``[text]`` when ``len(text) <= chunk_size``.
+    Returns ``[]`` for empty input.
+    Each chunk is at most ``chunk_size`` characters.
+    Consecutive chunks share ``chunk_overlap`` characters at their boundary.
+    """
+    if not text:
+        return []
+    if len(text) <= chunk_size:
+        return [text]
+
+    chunks: list[str] = []
+    step = max(1, chunk_size - chunk_overlap)
+    start = 0
+    while start < len(text):
+        end = start + chunk_size
+        chunks.append(text[start:end])
+        start += step
+    return chunks
+
+
+# ---------------------------------------------------------------------------
+# ASTChunker
+# ---------------------------------------------------------------------------
+
+
+class ASTChunker:
+    """Chunk code files at AST syntactic boundaries with a token fallback.
+
+    Implements ``ChunkerPort``.
+    Language is auto-detected from ``metadata["source"]`` file extension.
+    """
+
+    def __init__(self, settings: MemKernelSettings) -> None:
+        self._settings = settings
+        # Lazy cache: language name -> tree-sitter Parser | None
+        self._parsers: dict[str, Any] = {}
+
+    # ------------------------------------------------------------------
+    # ChunkerPort
+    # ------------------------------------------------------------------
+
+    def chunk(self, content: str, metadata: dict[str, Any]) -> list[str]:
+        """Return a list of chunk strings for *content*.
+
+        Uses AST chunking when the source language is supported and
+        tree-sitter is available; otherwise falls back to token chunking.
+        """
+        if not content:
+            return []
+
+        source: str = metadata.get("source", "")
+        ext = Path(source).suffix.lower()
+        language = _EXT_TO_LANG.get(ext)
+
+        if language is None:
+            logger.debug("Unknown extension %r — using token chunker", ext)
+            return _token_chunks(content, self._settings.chunk_size, self._settings.chunk_overlap)
+
+        ast_result = self._ast_chunks(content, language)
+        if ast_result is not None:
+            return ast_result
+
+        logger.debug("AST chunking produced no splits for %r — using token chunker", source)
+        return _token_chunks(content, self._settings.chunk_size, self._settings.chunk_overlap)
+
+    # ------------------------------------------------------------------
+    # AST chunking internals
+    # ------------------------------------------------------------------
+
+    def _ast_chunks(self, content: str, language: str) -> list[str] | None:
+        """Attempt AST-based chunking.  Returns None to signal fallback."""
+        parser = self._load_parser(language)
+        if parser is None:
+            return None
+
+        encoded = content.encode("utf-8")
+        try:
+            tree = parser.parse(encoded)
+        except Exception:
+            logger.debug("tree-sitter parse failed — falling back to token chunker", exc_info=True)
+            return None
+
+        boundary_types = _BOUNDARY_NODES.get(language, frozenset())
+        chunks: list[str] = []
+        self._walk(tree.root_node, encoded, boundary_types, chunks)
+
+        if not chunks:
+            return None
+        return chunks
+
+    def _walk(
+        self,
+        node: Any,
+        encoded: bytes,
+        boundary_types: frozenset[str],
+        chunks: list[str],
+    ) -> None:
+        """Depth-first walk; emit boundary nodes as chunks or recurse into them.
+
+        Uses the original UTF-8 byte sequence for slicing so that byte offsets
+        from tree-sitter nodes are always correct, even for non-ASCII source.
+        """
+        if node.type in boundary_types:
+            text = encoded[node.start_byte : node.end_byte].decode("utf-8")
+            if len(text) <= self._settings.chunk_size:
+                if text.strip():
+                    chunks.append(text)
+                return
+            # Oversized boundary node: collect all inner boundary nodes at any
+            # depth, then interleave the gap bytes (preamble, docstrings, class
+            # attributes — anything between inner boundaries) with recursive
+            # boundary chunk emission.  Gaps are token-chunked so no content is
+            # silently dropped regardless of the intermediate tree structure.
+            inner = sorted(
+                _inner_boundary_nodes(node, boundary_types),
+                key=lambda n: n.start_byte,
+            )
+            prev_end = node.start_byte
+            for inner_node in inner:
+                gap = encoded[prev_end : inner_node.start_byte].decode("utf-8")
+                if gap.strip():
+                    for part in _token_chunks(gap, self._settings.chunk_size, self._settings.chunk_overlap):
+                        if part.strip():
+                            chunks.append(part)
+                self._walk(inner_node, encoded, boundary_types, chunks)
+                prev_end = inner_node.end_byte
+            # Trailing gap (anything after the last inner boundary)
+            gap = encoded[prev_end : node.end_byte].decode("utf-8")
+            if gap.strip():
+                for part in _token_chunks(gap, self._settings.chunk_size, self._settings.chunk_overlap):
+                    if part.strip():
+                        chunks.append(part)
+            return
+
+        for child in node.children:
+            self._walk(child, encoded, boundary_types, chunks)
+
+    # ------------------------------------------------------------------
+    # Parser loader
+    # ------------------------------------------------------------------
+
+    def _load_parser(self, language: str) -> Any | None:
+        """Lazy-load and cache a tree-sitter Parser for *language*.
+
+        Returns None on any import/init error so the caller can fall back
+        gracefully.
+        """
+        if language in self._parsers:
+            return self._parsers[language]
+
+        parser: Any | None = None
+        try:
+            import importlib
+
+            import tree_sitter
+
+            module_name = _LANG_MODULE[language]
+            lang_module = importlib.import_module(module_name)
+
+            # tree-sitter-python (and siblings) expose a ``language()``
+            # function that returns a Language object.
+            lang_obj = tree_sitter.Language(lang_module.language())
+            p = tree_sitter.Parser(lang_obj)
+            parser = p
+            logger.debug("Loaded tree-sitter parser for %r", language)
+        except (ImportError, OSError, ValueError, Exception):
+            logger.debug(
+                "Could not load tree-sitter parser for %r — will use token chunker",
+                language,
+                exc_info=True,
+            )
+            parser = None
+
+        self._parsers[language] = parser
+        return parser

memkernel/retrieval/embeddings.py

@@ -0,0 +1,39 @@
+# Copyright (c) 2026 EGOGE
+# SPDX-License-Identifier: Apache-2.0
+"""Embedding adapter for MemKernel — sentence-transformers backend."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from memkernel.config import MemKernelSettings
+
+
+class SentenceTransformerEmbedding:
+    """EmbeddingPort implementation backed by sentence-transformers."""
+
+    def __init__(self, settings: MemKernelSettings) -> None:
+        # Lazy import: sentence_transformers is heavy; defer until first use.
+        from sentence_transformers import SentenceTransformer
+
+        self._model = SentenceTransformer(settings.embedding_model)
+        raw_dim = self._model.get_sentence_embedding_dimension()
+        if raw_dim is None:
+            raise ValueError(
+                f"Model '{settings.embedding_model}' returned None for embedding dimension. "
+                "Ensure the model has a known output dimension."
+            )
+        self._dimension: int = raw_dim
+
+    def embed(self, texts: list[str]) -> list[list[float]]:
+        """Return one embedding vector per input text."""
+        import numpy as np
+
+        raw = self._model.encode(texts, convert_to_numpy=True)
+        return [row.tolist() for row in np.atleast_2d(raw)]
+
+    @property
+    def dimension(self) -> int:
+        """Dimensionality of the embedding vectors."""
+        return self._dimension