keep-skill 0.1.0__py3-none-any.whl

keep/config.py

@@ -0,0 +1,323 @@
+"""
+Configuration management for associative memory stores.
+
+The configuration is stored as a TOML file in the store directory.
+It specifies which providers to use and their parameters.
+"""
+
+import os
+import platform
+import tomllib
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+# tomli_w for writing TOML (tomllib is read-only)
+try:
+    import tomli_w
+except ImportError:
+    tomli_w = None  # type: ignore
+
+
+CONFIG_FILENAME = "keep.toml"
+CONFIG_VERSION = 1
+
+
+@dataclass
+class ProviderConfig:
+    """Configuration for a single provider."""
+    name: str
+    params: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class StoreConfig:
+    """Complete store configuration."""
+    path: Path
+    version: int = CONFIG_VERSION
+    created: str = field(default_factory=lambda: datetime.now(timezone.utc).isoformat())
+    
+    # Provider configurations
+    embedding: ProviderConfig = field(default_factory=lambda: ProviderConfig("sentence-transformers"))
+    summarization: ProviderConfig = field(default_factory=lambda: ProviderConfig("truncate"))
+    document: ProviderConfig = field(default_factory=lambda: ProviderConfig("composite"))
+    
+    @property
+    def config_path(self) -> Path:
+        """Path to the TOML config file."""
+        return self.path / CONFIG_FILENAME
+    
+    def exists(self) -> bool:
+        """Check if config file exists."""
+        return self.config_path.exists()
+
+
+def read_openclaw_config() -> dict | None:
+    """
+    Read OpenClaw configuration if available.
+
+    Checks:
+    1. OPENCLAW_CONFIG environment variable
+    2. ~/.openclaw/openclaw.json (default location)
+
+    Returns None if not found or invalid.
+    """
+    import json
+
+    # Try environment variable first
+    config_path_str = os.environ.get("OPENCLAW_CONFIG")
+    if config_path_str:
+        config_file = Path(config_path_str)
+    else:
+        # Default location
+        config_file = Path.home() / ".openclaw" / "openclaw.json"
+
+    if not config_file.exists():
+        return None
+
+    try:
+        with open(config_file) as f:
+            return json.load(f)
+    except (json.JSONDecodeError, IOError):
+        return None
+
+
+def get_openclaw_memory_search_config(openclaw_config: dict | None) -> dict | None:
+    """
+    Extract memorySearch config from OpenClaw config.
+
+    Returns the memorySearch settings or None if not configured.
+
+    Example structure:
+        {
+            "provider": "openai" | "gemini" | "local" | "auto",
+            "model": "text-embedding-3-small",
+            "remote": {
+                "apiKey": "sk-...",
+                "baseUrl": "https://..."
+            }
+        }
+    """
+    if not openclaw_config:
+        return None
+
+    return (openclaw_config
+            .get("agents", {})
+            .get("defaults", {})
+            .get("memorySearch", None))
+
+
+def detect_default_providers() -> dict[str, ProviderConfig]:
+    """
+    Detect the best default providers for the current environment.
+
+    Priority for embeddings:
+    1. OpenClaw memorySearch config (if configured with provider + API key)
+    2. sentence-transformers (local fallback)
+
+    Priority for summarization:
+    1. OpenClaw model config + Anthropic (if configured and ANTHROPIC_API_KEY available)
+    2. MLX (Apple Silicon local-first)
+    3. OpenAI (if API key available)
+    4. Fallback: truncate
+
+    Returns provider configs for: embedding, summarization, document
+    """
+    providers = {}
+
+    # Check for Apple Silicon
+    is_apple_silicon = (
+        platform.system() == "Darwin" and
+        platform.machine() == "arm64"
+    )
+
+    # Check for API keys
+    has_anthropic_key = bool(os.environ.get("ANTHROPIC_API_KEY"))
+    has_openai_key = bool(
+        os.environ.get("KEEP_OPENAI_API_KEY") or
+        os.environ.get("OPENAI_API_KEY")
+    )
+    has_gemini_key = bool(
+        os.environ.get("GEMINI_API_KEY") or
+        os.environ.get("GOOGLE_API_KEY")
+    )
+
+    # Check for OpenClaw config
+    openclaw_config = read_openclaw_config()
+    openclaw_model = None
+    if openclaw_config:
+        model_str = (openclaw_config.get("agents", {})
+                     .get("defaults", {})
+                     .get("model", {})
+                     .get("primary", ""))
+        if model_str:
+            openclaw_model = model_str
+
+    # Get OpenClaw memorySearch config for embeddings
+    memory_search = get_openclaw_memory_search_config(openclaw_config)
+
+    # Embedding: check OpenClaw memorySearch config first, then fall back to local
+    embedding_provider = None
+    if memory_search:
+        ms_provider = memory_search.get("provider", "auto")
+        ms_model = memory_search.get("model")
+        ms_api_key = memory_search.get("remote", {}).get("apiKey")
+
+        if ms_provider == "openai" or (ms_provider == "auto" and has_openai_key):
+            # Use OpenAI embeddings if configured or auto with key available
+            api_key = ms_api_key or os.environ.get("OPENAI_API_KEY")
+            if api_key:
+                params = {}
+                if ms_model:
+                    params["model"] = ms_model
+                embedding_provider = ProviderConfig("openai", params)
+
+        elif ms_provider == "gemini" or (ms_provider == "auto" and has_gemini_key and not has_openai_key):
+            # Use Gemini embeddings if configured or auto with key available
+            api_key = ms_api_key or os.environ.get("GEMINI_API_KEY") or os.environ.get("GOOGLE_API_KEY")
+            if api_key:
+                params = {}
+                if ms_model:
+                    params["model"] = ms_model
+                embedding_provider = ProviderConfig("gemini", params)
+
+    # Fall back to sentence-transformers (local, always works)
+    if embedding_provider is None:
+        embedding_provider = ProviderConfig("sentence-transformers")
+
+    providers["embedding"] = embedding_provider
+    
+    # Summarization: priority order based on availability
+    # 1. OpenClaw + Anthropic (if configured and key available)
+    if openclaw_model and openclaw_model.startswith("anthropic/") and has_anthropic_key:
+        # Extract model name from "anthropic/claude-sonnet-4-5" format
+        model_name = openclaw_model.split("/", 1)[1] if "/" in openclaw_model else "claude-3-5-haiku-20241022"
+        # Map OpenClaw model names to actual Anthropic model names
+        model_mapping = {
+            "claude-sonnet-4": "claude-sonnet-4-20250514",
+            "claude-sonnet-4-5": "claude-sonnet-4-20250514",
+            "claude-sonnet-3-5": "claude-3-5-sonnet-20241022",
+            "claude-haiku-3-5": "claude-3-5-haiku-20241022",
+        }
+        actual_model = model_mapping.get(model_name, "claude-3-5-haiku-20241022")
+        providers["summarization"] = ProviderConfig("anthropic", {"model": actual_model})
+    # 2. MLX on Apple Silicon (local-first)
+    elif is_apple_silicon:
+        try:
+            import mlx_lm  # noqa
+            providers["summarization"] = ProviderConfig("mlx", {"model": "mlx-community/Llama-3.2-3B-Instruct-4bit"})
+        except ImportError:
+            if has_openai_key:
+                providers["summarization"] = ProviderConfig("openai")
+            else:
+                providers["summarization"] = ProviderConfig("passthrough")
+    # 3. OpenAI (if key available)
+    elif has_openai_key:
+        providers["summarization"] = ProviderConfig("openai")
+    # 4. Fallback: truncate
+    else:
+        providers["summarization"] = ProviderConfig("truncate")
+    
+    # Document provider is always composite
+    providers["document"] = ProviderConfig("composite")
+    
+    return providers
+
+
+def create_default_config(store_path: Path) -> StoreConfig:
+    """Create a new config with auto-detected defaults."""
+    providers = detect_default_providers()
+    
+    return StoreConfig(
+        path=store_path,
+        embedding=providers["embedding"],
+        summarization=providers["summarization"],
+        document=providers["document"],
+    )
+
+
+def load_config(store_path: Path) -> StoreConfig:
+    """
+    Load configuration from a store directory.
+    
+    Raises:
+        FileNotFoundError: If config doesn't exist
+        ValueError: If config is invalid
+    """
+    config_path = store_path / CONFIG_FILENAME
+    
+    if not config_path.exists():
+        raise FileNotFoundError(f"Config not found: {config_path}")
+    
+    with open(config_path, "rb") as f:
+        data = tomllib.load(f)
+    
+    # Validate version
+    version = data.get("store", {}).get("version", 1)
+    if version > CONFIG_VERSION:
+        raise ValueError(f"Config version {version} is newer than supported ({CONFIG_VERSION})")
+    
+    # Parse provider configs
+    def parse_provider(section: dict) -> ProviderConfig:
+        return ProviderConfig(
+            name=section.get("name", ""),
+            params={k: v for k, v in section.items() if k != "name"},
+        )
+    
+    return StoreConfig(
+        path=store_path,
+        version=version,
+        created=data.get("store", {}).get("created", ""),
+        embedding=parse_provider(data.get("embedding", {"name": "sentence-transformers"})),
+        summarization=parse_provider(data.get("summarization", {"name": "truncate"})),
+        document=parse_provider(data.get("document", {"name": "composite"})),
+    )
+
+
+def save_config(config: StoreConfig) -> None:
+    """
+    Save configuration to the store directory.
+    
+    Creates the directory if it doesn't exist.
+    """
+    if tomli_w is None:
+        raise RuntimeError("tomli_w is required to save config. Install with: pip install tomli-w")
+    
+    # Ensure directory exists
+    config.path.mkdir(parents=True, exist_ok=True)
+    
+    # Build TOML structure
+    def provider_to_dict(p: ProviderConfig) -> dict:
+        d = {"name": p.name}
+        d.update(p.params)
+        return d
+    
+    data = {
+        "store": {
+            "version": config.version,
+            "created": config.created,
+        },
+        "embedding": provider_to_dict(config.embedding),
+        "summarization": provider_to_dict(config.summarization),
+        "document": provider_to_dict(config.document),
+    }
+    
+    with open(config.config_path, "wb") as f:
+        tomli_w.dump(data, f)
+
+
+def load_or_create_config(store_path: Path) -> StoreConfig:
+    """
+    Load existing config or create a new one with defaults.
+    
+    This is the main entry point for config management.
+    """
+    config_path = store_path / CONFIG_FILENAME
+    
+    if config_path.exists():
+        return load_config(store_path)
+    else:
+        config = create_default_config(store_path)
+        save_config(config)
+        return config

keep/context.py

@@ -0,0 +1,127 @@
+"""
+Working context and top-of-mind retrieval.
+
+This module provides hierarchical context management for efficient
+"what are we working on?" queries with O(log(log(N))) retrieval.
+"""
+
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from typing import Any, Optional
+
+
+@dataclass
+class WorkingContext:
+    """
+    The current working context — a high-level summary of active work.
+    
+    This is the "Level 3" summary that any agent can read to instantly
+    understand what's being worked on.
+    
+    Attributes:
+        summary: Natural language description of current focus
+        active_items: IDs of items currently being worked with
+        topics: Active topic/domain tags
+        updated: When context was last updated
+        session_id: Current session identifier
+        metadata: Additional context-specific data (arbitrary structure)
+    """
+    summary: str
+    active_items: list[str] = field(default_factory=list)
+    topics: list[str] = field(default_factory=list)
+    updated: str = field(default_factory=lambda: datetime.now(timezone.utc).isoformat())
+    session_id: Optional[str] = None
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass  
+class TopicSummary:
+    """
+    A summary of items within a topic cluster (Level 2).
+    
+    Topics aggregate related items and provide a mid-level
+    overview without retrieving all underlying items.
+    
+    Attributes:
+        topic: Topic identifier (tag value)
+        summary: Generated summary of topic contents
+        item_count: Number of items in this topic
+        key_items: IDs of the most important items in the topic
+        subtopics: Child topics if hierarchical
+        updated: When topic summary was last regenerated
+    """
+    topic: str
+    summary: str
+    item_count: int
+    key_items: list[str] = field(default_factory=list)
+    subtopics: list[str] = field(default_factory=list)
+    updated: str = field(default_factory=lambda: datetime.now(timezone.utc).isoformat())
+
+
+@dataclass
+class RoutingContext:
+    """
+    Describes how items are routed between private and shared stores.
+    
+    This document lives at a well-known location in the shared store.
+    The facade reads it to make routing decisions. The private store
+    is physically separate and invisible from the shared store.
+    
+    Attributes:
+        summary: Natural language description of the privacy model
+        private_patterns: Tag patterns that route to private store (each pattern is dict[str, str])
+        private_store_path: Location of the private store (if local)
+        updated: When routing was last modified
+        metadata: Additional routing configuration
+    """
+    summary: str = "Items tagged for private/draft visibility route to a separate store."
+    private_patterns: list[dict[str, str]] = field(default_factory=lambda: [
+        {"_visibility": "draft"},
+        {"_visibility": "private"},
+        {"_for": "self"},
+    ])
+    private_store_path: Optional[str] = None  # Resolved at init; None = default location
+    updated: str = field(default_factory=lambda: datetime.now(timezone.utc).isoformat())
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+
+# Well-known item ID for the routing context document
+ROUTING_CONTEXT_ID = "_system:routing"
+
+
+# Reserved system tags for context management (stored with items)
+CONTEXT_TAGS = {
+    "_session": "Session that last touched this item",
+    "_topic": "Primary topic classification",
+    "_level": "Hierarchy level (0=source, 1=cluster, 2=topic, 3=context)",
+    "_summarizes": "IDs of items this item summarizes (for hierarchy)",
+}
+
+# Relevance scoring is computed at query time, NOT stored.
+# This preserves agility between broad exploration and focused work.
+# Score factors:
+#   - semantic similarity to query/hint
+#   - recency (time decay)
+#   - topic overlap with current WorkingContext.topics
+#   - session affinity (same session = boost)
+# The weighting of these factors can vary by retrieval mode.
+
+
+def generate_session_id() -> str:
+    """Generate a unique session identifier."""
+    import uuid
+    date = datetime.now(timezone.utc).strftime("%Y-%m-%d")
+    short_uuid = uuid.uuid4().hex[:8]
+    return f"{date}:{short_uuid}"
+
+
+def matches_private_pattern(tags: dict[str, str], patterns: list[dict[str, str]]) -> bool:
+    """
+    Check if an item's tags match any private routing pattern.
+    
+    A pattern matches if ALL its key-value pairs are present in tags.
+    """
+    for pattern in patterns:
+        if all(tags.get(k) == v for k, v in pattern.items()):
+            return True
+    return False

keep/indexing.py

@@ -0,0 +1,208 @@
+"""
+Indexing modes for controlling embedding granularity.
+
+Summarization ALWAYS happens (it's cheap and aids retrieval).
+The mode controls what gets embedded:
+
+- DOCUMENT: Embed summary only (1 vector per doc, fast)
+- CHUNKED: Embed chunks only (N vectors per doc, OpenClaw-compatible)  
+- HYBRID: Embed summary + chunks (best recall, more storage)
+- BM25_ONLY: Fulltext index only (no embeddings, keyword search)
+"""
+
+from dataclasses import dataclass
+from enum import Enum
+from typing import Iterator, Protocol
+
+
+class IndexingMode(Enum):
+    """Controls embedding granularity. Summary is always stored."""
+    
+    DOCUMENT = "document"
+    """
+    Embed summary only.
+    - One vector per document
+    - Fast, good for "what is this document about?"
+    - Summary always available for display
+    """
+    
+    CHUNKED = "chunked"
+    """
+    Embed chunks only.
+    - N vectors per document (one per ~400-token chunk)
+    - OpenClaw-compatible mode
+    - Good for passage-level retrieval
+    - Summary stored but not embedded
+    """
+    
+    HYBRID = "hybrid"
+    """
+    Embed summary AND chunks.
+    - 1+N vectors per document
+    - Best recall (semantic anchor + passage-level)
+    - More storage, more embedding calls
+    """
+    
+    BM25_ONLY = "bm25_only"
+    """
+    Fulltext index only.
+    - No embeddings at all
+    - Summary stored for display
+    - Keyword search only (exact token matching)
+    - Fastest, minimal resource usage
+    """
+
+
+@dataclass
+class IndexingConfig:
+    """Configuration for the indexing pipeline."""
+    
+    mode: IndexingMode = IndexingMode.DOCUMENT
+    """Which embedding strategy to use. Summary always stored."""
+    
+    # Chunking settings (for CHUNKED/HYBRID modes)
+    chunk_target_tokens: int = 400
+    """Target tokens per chunk (OpenClaw default: 400)."""
+    
+    chunk_overlap_tokens: int = 80
+    """Overlap between chunks (OpenClaw default: 80)."""
+    
+    tokens_per_word: float = 1.3
+    """Approximation for token estimation."""
+    
+    # Summarization settings (always used)
+    summary_max_chars: int = 500
+    """Maximum summary length in characters."""
+    
+    # BM25 settings
+    enable_fulltext: bool = True
+    """Whether to build FTS index alongside vectors."""
+    
+    # Hybrid search weights (vector + BM25)
+    vector_weight: float = 0.7
+    """Weight for vector similarity in hybrid search."""
+    
+    text_weight: float = 0.3
+    """Weight for BM25 score in hybrid search."""
+    
+    @classmethod
+    def document_mode(cls) -> "IndexingConfig":
+        """Fast: embed summary only."""
+        return cls(mode=IndexingMode.DOCUMENT)
+    
+    @classmethod
+    def chunked_mode(cls) -> "IndexingConfig":
+        """OpenClaw-compatible: embed chunks."""
+        return cls(
+            mode=IndexingMode.CHUNKED,
+            chunk_target_tokens=400,
+            chunk_overlap_tokens=80,
+            enable_fulltext=True,
+            vector_weight=0.7,
+            text_weight=0.3,
+        )
+    
+    @classmethod
+    def hybrid_mode(cls) -> "IndexingConfig":
+        """Best recall: embed summary + chunks."""
+        return cls(
+            mode=IndexingMode.HYBRID,
+            chunk_target_tokens=400,
+            chunk_overlap_tokens=80,
+        )
+    
+    @classmethod
+    def bm25_only(cls) -> "IndexingConfig":
+        """Fastest: no embeddings, keyword search only."""
+        return cls(
+            mode=IndexingMode.BM25_ONLY,
+            enable_fulltext=True,
+        )
+    
+    def __post_init__(self):
+        # Normalize weights
+        total = self.vector_weight + self.text_weight
+        if total > 0:
+            self.vector_weight = self.vector_weight / total
+            self.text_weight = self.text_weight / total
+
+
+# --- Chunking ---
+
+@dataclass(frozen=True)
+class Chunk:
+    """A chunk of text with position info."""
+    text: str
+    start_char: int
+    end_char: int
+    index: int  # 0-based chunk number
+
+
+class Chunker(Protocol):
+    """Protocol for text chunking strategies."""
+    
+    def chunk(self, text: str) -> Iterator[Chunk]:
+        """Split text into overlapping chunks."""
+        ...
+
+
+@dataclass
+class TokenChunker:
+    """Chunk by approximate token count with overlap.
+    
+    OpenClaw defaults: ~400 tokens target, 80 token overlap.
+    """
+    target_tokens: int = 400
+    overlap_tokens: int = 80
+    tokens_per_word: float = 1.3
+    
+    def chunk(self, text: str) -> Iterator[Chunk]:
+        """Split text into overlapping chunks by token estimate."""
+        if not text.strip():
+            return
+        
+        words = text.split()
+        if not words:
+            return
+            
+        target_words = int(self.target_tokens / self.tokens_per_word)
+        overlap_words = int(self.overlap_tokens / self.tokens_per_word)
+        step_words = max(1, target_words - overlap_words)
+        
+        # Track character positions
+        word_positions: list[tuple[int, int]] = []
+        pos = 0
+        for word in words:
+            start = text.find(word, pos)
+            end = start + len(word)
+            word_positions.append((start, end))
+            pos = end
+        
+        chunk_index = 0
+        word_index = 0
+        
+        while word_index < len(words):
+            end_word = min(word_index + target_words, len(words))
+            chunk_words = words[word_index:end_word]
+            
+            start_char = word_positions[word_index][0]
+            end_char = word_positions[end_word - 1][1]
+            
+            yield Chunk(
+                text=" ".join(chunk_words),
+                start_char=start_char,
+                end_char=end_char,
+                index=chunk_index,
+            )
+            
+            chunk_index += 1
+            word_index += step_words
+            
+            # Don't create tiny final chunks
+            if word_index < len(words) and len(words) - word_index < overlap_words:
+                break
+
+
+def estimate_tokens(text: str, tokens_per_word: float = 1.3) -> int:
+    """Estimate token count from text."""
+    return int(len(text.split()) * tokens_per_word)