echostate 0.1.0__py3-none-any.whl

echostate/__init__.py

@@ -0,0 +1,27 @@
+"""
+EchoState - Semantic, event-sourced state for intelligent systems
+"""
+
+from echostate.state import EchoState
+from echostate.event import Event
+from echostate.embeddings import SearchHit
+from echostate.exceptions import (
+    EchoStateError,
+    EchoStateLockedError,
+    EchoStateSerializationError,
+    EchoStatePathError,
+    EchoStateEmbeddingError,
+)
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "EchoState",
+    "Event",
+    "SearchHit",
+    "EchoStateError",
+    "EchoStateLockedError",
+    "EchoStateSerializationError",
+    "EchoStatePathError",
+    "EchoStateEmbeddingError",
+]

echostate/database.py

@@ -0,0 +1,413 @@
+"""SQLite database management for EchoState."""
+
+import sqlite3
+import json
+from pathlib import Path
+from typing import Optional, List, Dict, Any
+from contextlib import contextmanager
+
+from echostate.exceptions import EchoStateLockedError
+from echostate.event import Event
+
+
+class Database:
+    """Manages SQLite database for EchoState."""
+
+    def __init__(self, db_path: str):
+        """
+        Initialize database connection.
+
+        Args:
+            db_path: SQLite database path (e.g., "sqlite:///state.db" or "state.db")
+        """
+        # Parse sqlite:/// prefix if present
+        if db_path.startswith("sqlite:///"):
+            db_path = db_path[10:]  # Remove "sqlite:///"
+        elif db_path.startswith("sqlite://"):
+            db_path = db_path[9:]  # Remove "sqlite://"
+
+        self.db_path = Path(db_path)
+        self._conn: Optional[sqlite3.Connection] = None
+        self._ensure_schema()
+
+    def _ensure_schema(self):
+        """Create database schema if it doesn't exist."""
+        conn = self._get_connection()
+        cursor = conn.cursor()
+
+        # Events table
+        cursor.execute("""
+            CREATE TABLE IF NOT EXISTS events (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                timestamp INTEGER NOT NULL,
+                path TEXT NOT NULL,
+                operation TEXT NOT NULL,
+                value TEXT,
+                event_version INTEGER NOT NULL DEFAULT 1,
+                metadata TEXT
+            )
+        """)
+
+        # Snapshots table
+        cursor.execute("""
+            CREATE TABLE IF NOT EXISTS snapshots (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                last_event_id INTEGER NOT NULL,
+                snapshot TEXT NOT NULL,
+                timestamp INTEGER NOT NULL
+            )
+        """)
+
+        # Metadata table (for EchoState instance metadata)
+        cursor.execute("""
+            CREATE TABLE IF NOT EXISTS metadata (
+                key TEXT PRIMARY KEY,
+                value TEXT NOT NULL
+            )
+        """)
+
+        # Embeddings table (for semantic index)
+        cursor.execute("""
+            CREATE TABLE IF NOT EXISTS embeddings (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                event_id INTEGER NOT NULL,
+                path TEXT NOT NULL,
+                text TEXT NOT NULL,
+                embedding BLOB NOT NULL,
+                model_id TEXT NOT NULL,
+                metadata TEXT
+            )
+        """)
+
+        # Indexes
+        cursor.execute("""
+            CREATE INDEX IF NOT EXISTS idx_events_timestamp ON events(timestamp)
+        """)
+        cursor.execute("""
+            CREATE INDEX IF NOT EXISTS idx_events_path ON events(path)
+        """)
+        cursor.execute("""
+            CREATE INDEX IF NOT EXISTS idx_embeddings_event_id ON embeddings(event_id)
+        """)
+        cursor.execute("""
+            CREATE INDEX IF NOT EXISTS idx_embeddings_path ON embeddings(path)
+        """)
+
+        conn.commit()
+
+    def _get_connection(self) -> sqlite3.Connection:
+        """Get or create database connection."""
+        if self._conn is None:
+            try:
+                self._conn = sqlite3.connect(
+                    str(self.db_path),
+                    check_same_thread=False,
+                    timeout=5.0,
+                )
+                self._conn.row_factory = sqlite3.Row
+            except sqlite3.OperationalError as e:
+                if "locked" in str(e).lower():
+                    raise EchoStateLockedError(
+                        f"Database is locked: {self.db_path}"
+                    ) from e
+                raise
+        return self._conn
+
+    @contextmanager
+    def transaction(self):
+        """Context manager for database transactions."""
+        conn = self._get_connection()
+        try:
+            yield conn
+            conn.commit()
+        except Exception:
+            conn.rollback()
+            raise
+
+    def append_event(self, event: Event) -> int:
+        """
+        Append an event to the event log.
+
+        Args:
+            event: Event to append
+
+        Returns:
+            The event ID assigned by the database
+        """
+        with self.transaction() as conn:
+            cursor = conn.cursor()
+            event_dict = event.to_dict()
+            cursor.execute("""
+                INSERT INTO events (timestamp, path, operation, value, event_version, metadata)
+                VALUES (:timestamp, :path, :operation, :value, :event_version, :metadata)
+            """, event_dict)
+            return cursor.lastrowid
+
+    def get_events(
+        self,
+        path: Optional[str] = None,
+        since_event_id: Optional[int] = None,
+        limit: Optional[int] = None,
+        order_desc: bool = False,
+    ) -> List[Event]:
+        """
+        Retrieve events from the event log.
+
+        Args:
+            path: Optional path filter (exact match)
+            since_event_id: Only return events with id > since_event_id
+            limit: Maximum number of events to return
+            order_desc: If True, order by id descending (most recent first)
+
+        Returns:
+            List of events, ordered by id (ascending by default, descending if order_desc=True)
+        """
+        conn = self._get_connection()
+        cursor = conn.cursor()
+
+        query = "SELECT * FROM events WHERE 1=1"
+        params = []
+
+        if path is not None:
+            query += " AND path = ?"
+            params.append(path)
+
+        if since_event_id is not None:
+            query += " AND id > ?"
+            params.append(since_event_id)
+
+        query += f" ORDER BY id {'DESC' if order_desc else 'ASC'}"
+
+        if limit is not None:
+            query += " LIMIT ?"
+            params.append(limit)
+
+        cursor.execute(query, params)
+        rows = cursor.fetchall()
+        return [Event.from_dict(dict(row)) for row in rows]
+
+    def get_events_with_metadata_filter(
+        self,
+        path: Optional[str] = None,
+        metadata_filters: Optional[Dict[str, Any]] = None,
+        limit: Optional[int] = None,
+    ) -> List[Event]:
+        """
+        Retrieve events with metadata filtering (Python-side filtering in v0.1).
+
+        Args:
+            path: Optional path filter (exact match)
+            metadata_filters: Dict of metadata fields to match
+            limit: Maximum number of events to return (most recent first)
+
+        Returns:
+            List of events matching filters, ordered by id descending
+        """
+        # Get all events matching path (ordered descending for most recent first)
+        events = self.get_events(path=path, limit=None, order_desc=True)
+
+        # Apply metadata filters in Python (v0.1 approach)
+        if metadata_filters:
+            filtered = []
+            for event in events:
+                if event.metadata:
+                    match = True
+                    for key, value in metadata_filters.items():
+                        if event.metadata.get(key) != value:
+                            match = False
+                            break
+                    if match:
+                        filtered.append(event)
+                else:
+                    # Event has no metadata, skip if filters are specified
+                    continue
+            events = filtered
+
+        # Apply limit after filtering
+        if limit is not None:
+            events = events[:limit]
+
+        return events
+
+    def get_snapshot_at_event_id(self, event_id: int) -> Optional[Dict[str, Any]]:
+        """
+        Get the latest snapshot that includes events up to the given event_id.
+
+        Args:
+            event_id: Target event ID
+
+        Returns:
+            Snapshot dict with keys: id, last_event_id, snapshot, timestamp
+            or None if no suitable snapshot exists
+        """
+        conn = self._get_connection()
+        cursor = conn.cursor()
+        cursor.execute("""
+            SELECT * FROM snapshots
+            WHERE last_event_id <= ?
+            ORDER BY last_event_id DESC
+            LIMIT 1
+        """, (event_id,))
+        row = cursor.fetchone()
+        if row is None:
+            return None
+        result = dict(row)
+        result["snapshot"] = json.loads(result["snapshot"])
+        return result
+
+    def get_events_up_to(self, event_id: int, since_event_id: Optional[int] = None) -> List[Event]:
+        """
+        Get events up to and including a specific event_id.
+
+        Args:
+            event_id: Maximum event ID to include
+            since_event_id: Only return events with id > since_event_id
+
+        Returns:
+            List of events ordered by id ascending
+        """
+        conn = self._get_connection()
+        cursor = conn.cursor()
+
+        query = "SELECT * FROM events WHERE id <= ?"
+        params = [event_id]
+
+        if since_event_id is not None:
+            query += " AND id > ?"
+            params.append(since_event_id)
+
+        query += " ORDER BY id ASC"
+
+        cursor.execute(query, params)
+        rows = cursor.fetchall()
+        return [Event.from_dict(dict(row)) for row in rows]
+
+    def store_embedding(
+        self,
+        event_id: int,
+        path: str,
+        text: str,
+        embedding: bytes,
+        model_id: str,
+        metadata: Optional[str] = None,
+    ) -> int:
+        """
+        Store an embedding for an event.
+
+        Args:
+            event_id: Associated event ID
+            path: Event path
+            text: Derived record text
+            embedding: Embedding as bytes
+            model_id: Model identifier
+            metadata: Optional metadata JSON string
+
+        Returns:
+            Embedding ID
+        """
+        with self.transaction() as conn:
+            cursor = conn.cursor()
+            cursor.execute("""
+                INSERT INTO embeddings (event_id, path, text, embedding, model_id, metadata)
+                VALUES (?, ?, ?, ?, ?, ?)
+            """, (event_id, path, text, embedding, model_id, metadata))
+            return cursor.lastrowid
+
+    def get_all_embeddings(self, model_id: Optional[str] = None) -> List[Dict[str, Any]]:
+        """
+        Get all embeddings, optionally filtered by model_id.
+
+        Args:
+            model_id: Optional model ID filter
+
+        Returns:
+            List of embedding records as dicts
+        """
+        conn = self._get_connection()
+        cursor = conn.cursor()
+
+        if model_id:
+            cursor.execute("""
+                SELECT * FROM embeddings
+                WHERE model_id = ?
+                ORDER BY id ASC
+            """, (model_id,))
+        else:
+            cursor.execute("""
+                SELECT * FROM embeddings
+                ORDER BY id ASC
+            """)
+
+        rows = cursor.fetchall()
+        return [dict(row) for row in rows]
+
+    def truncate_embeddings(self, model_id: Optional[str] = None):
+        """
+        Delete all embeddings, optionally filtered by model_id.
+
+        Args:
+            model_id: If provided, only delete embeddings for this model
+        """
+        with self.transaction() as conn:
+            cursor = conn.cursor()
+            if model_id:
+                cursor.execute("DELETE FROM embeddings WHERE model_id = ?", (model_id,))
+            else:
+                cursor.execute("DELETE FROM embeddings")
+
+    def get_latest_snapshot(self) -> Optional[Dict[str, Any]]:
+        """
+        Get the latest snapshot.
+
+        Returns:
+            Dict with keys: id, last_event_id, snapshot (parsed JSON), timestamp
+            or None if no snapshot exists
+        """
+        conn = self._get_connection()
+        cursor = conn.cursor()
+        cursor.execute("""
+            SELECT * FROM snapshots
+            ORDER BY last_event_id DESC
+            LIMIT 1
+        """)
+        row = cursor.fetchone()
+        if row is None:
+            return None
+        result = dict(row)
+        result["snapshot"] = json.loads(result["snapshot"])
+        return result
+
+    def create_snapshot(self, last_event_id: int, snapshot: Dict[str, Any]) -> int:
+        """
+        Create a new snapshot.
+
+        Args:
+            last_event_id: The last event ID included in this snapshot
+            snapshot: The state snapshot as a dictionary
+
+        Returns:
+            The snapshot ID
+        """
+        with self.transaction() as conn:
+            cursor = conn.cursor()
+            import time
+            timestamp = int(time.time() * 1000)
+            cursor.execute("""
+                INSERT INTO snapshots (last_event_id, snapshot, timestamp)
+                VALUES (?, ?, ?)
+            """, (last_event_id, json.dumps(snapshot), timestamp))
+            return cursor.lastrowid
+
+    def close(self):
+        """Close database connection."""
+        if self._conn is not None:
+            self._conn.close()
+            self._conn = None
+
+    def __enter__(self):
+        """Context manager entry."""
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        """Context manager exit."""
+        self.close()

echostate/embeddings.py

@@ -0,0 +1,143 @@
+"""Embedding generation and storage for semantic search."""
+
+import json
+import numpy as np
+from dataclasses import dataclass
+from typing import List, Optional, Dict, Any, Tuple
+from sentence_transformers import SentenceTransformer
+
+from echostate.event import Event
+from echostate.exceptions import EchoStateEmbeddingError
+
+
+class EmbeddingIndex:
+    """Manages embeddings for semantic search."""
+
+    def __init__(self, model_id: str, model_name: str):
+        """
+        Initialize embedding index.
+
+        Args:
+            model_id: Model identifier (e.g., "st:all-MiniLM-L6-v2")
+            model_name: Model name for sentence-transformers
+        """
+        self.model_id = model_id
+        self.model_name = model_name
+        self._model: Optional[SentenceTransformer] = None
+
+    def _get_model(self) -> SentenceTransformer:
+        """Lazy-load the embedding model."""
+        if self._model is None:
+            try:
+                self._model = SentenceTransformer(self.model_name)
+            except Exception as e:
+                raise EchoStateEmbeddingError(
+                    f"Failed to load embedding model '{self.model_name}': {e}"
+                ) from e
+        return self._model
+
+    def create_derived_record(self, event: Event) -> Optional[str]:
+        """
+        Create a derived record (text representation) from an event.
+
+        Args:
+            event: Event to convert
+
+        Returns:
+            Text representation suitable for embedding, or None if event shouldn't be indexed
+        """
+        # Deletes don't produce searchable content
+        if event.operation == "delete":
+            return None
+
+        value = event.value
+
+        # Convert value to text based on type
+        if isinstance(value, str):
+            text = value
+        elif isinstance(value, (dict, list)):
+            # Serialize to compact JSON
+            text = json.dumps(value, separators=(",", ":"), ensure_ascii=False)
+        elif value is None:
+            return None
+        else:
+            # Primitives
+            text = str(value)
+
+        # For append operations, add lightweight context
+        if event.operation == "append":
+            # Include path context
+            text = f"{event.path}: {text}"
+
+        return text
+
+    def generate_embedding(self, text: str) -> bytes:
+        """
+        Generate embedding for a text string.
+
+        Args:
+            text: Text to embed
+
+        Returns:
+            Embedding as bytes (numpy array serialized)
+        """
+        model = self._get_model()
+        try:
+            embedding = model.encode(text, convert_to_numpy=True)
+            # Convert to bytes for storage
+            return embedding.tobytes()
+        except Exception as e:
+            raise EchoStateEmbeddingError(f"Failed to generate embedding: {e}") from e
+
+    def compute_similarity(self, query_embedding: bytes, target_embedding: bytes) -> float:
+        """
+        Compute cosine similarity between two embeddings.
+
+        Args:
+            query_embedding: Query embedding as bytes
+            target_embedding: Target embedding as bytes
+
+        Returns:
+            Cosine similarity score (0-1, higher is more similar)
+        """
+        query_vec = np.frombuffer(query_embedding, dtype=np.float32)
+        target_vec = np.frombuffer(target_embedding, dtype=np.float32)
+
+        # Cosine similarity
+        dot_product = np.dot(query_vec, target_vec)
+        norm_query = np.linalg.norm(query_vec)
+        norm_target = np.linalg.norm(target_vec)
+
+        if norm_query == 0 or norm_target == 0:
+            return 0.0
+
+        similarity = dot_product / (norm_query * norm_target)
+        # Normalize to 0-1 range (cosine similarity is -1 to 1)
+        normalized = (similarity + 1) / 2
+        # Convert numpy scalar to Python float
+        return float(normalized)
+
+    def embed_query(self, query: str) -> bytes:
+        """
+        Generate embedding for a search query.
+
+        Args:
+            query: Search query string
+
+        Returns:
+            Embedding as bytes
+        """
+        return self.generate_embedding(query)
+
+
+@dataclass
+class SearchHit:
+    """Represents a search result hit."""
+
+    event_id: int
+    score: float
+    path: str
+    operation: str
+    text: str
+    metadata: Optional[Dict[str, Any]] = None
+    value: Optional[Any] = None  # Best-effort, may be None

echostate/event.py

@@ -0,0 +1,43 @@
+"""Event model for EchoState."""
+
+import json
+from dataclasses import dataclass, field
+from typing import Any, Dict, Optional
+from datetime import datetime
+
+
+@dataclass
+class Event:
+    """Represents a single state mutation event."""
+
+    id: Optional[int] = None  # Set by database on insert
+    timestamp: int = field(default_factory=lambda: int(datetime.now().timestamp() * 1000))
+    path: str = ""
+    operation: str = ""  # set | append | delete | update
+    value: Any = None
+    event_version: int = 1
+    metadata: Optional[Dict[str, Any]] = None
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert event to dictionary for database storage."""
+        return {
+            "timestamp": self.timestamp,
+            "path": self.path,
+            "operation": self.operation,
+            "value": json.dumps(self.value) if self.value is not None else None,
+            "event_version": self.event_version,
+            "metadata": json.dumps(self.metadata) if self.metadata else None,
+        }
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "Event":
+        """Create event from database row."""
+        return cls(
+            id=data.get("id"),
+            timestamp=data["timestamp"],
+            path=data["path"],
+            operation=data["operation"],
+            value=json.loads(data["value"]) if data["value"] else None,
+            event_version=data.get("event_version", 1),
+            metadata=json.loads(data["metadata"]) if data.get("metadata") else None,
+        )

echostate/exceptions.py

@@ -0,0 +1,31 @@
+"""Custom exceptions for EchoState."""
+
+
+class EchoStateError(Exception):
+    """Base exception for all EchoState errors."""
+
+    pass
+
+
+class EchoStateLockedError(EchoStateError):
+    """Raised when SQLite database is locked by another process."""
+
+    pass
+
+
+class EchoStateSerializationError(EchoStateError):
+    """Raised when a value cannot be JSON-serialized."""
+
+    pass
+
+
+class EchoStatePathError(EchoStateError):
+    """Raised when a path operation fails (e.g., setting a child under a non-dict)."""
+
+    pass
+
+
+class EchoStateEmbeddingError(EchoStateError):
+    """Raised when embedding generation fails (only for synchronous indexing)."""
+
+    pass